notebooks/Text Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import string\n",
    "import contractions\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "import re\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords, wordnet\n",
    "from nltk.stem import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./for_preprocessing.csv\") as f:\n",
    "    data = pd.read_csv(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>author</th>\n",
       "      <th>score</th>\n",
       "      <th>publish_date</th>\n",
       "      <th>num_comment</th>\n",
       "      <th>permalink</th>\n",
       "      <th>flair</th>\n",
       "      <th>selftext</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>abd9dz</td>\n",
       "      <td>Allow banks to hold passports of loan-takers, ...</td>\n",
       "      <td>https://www.reddit.com/r/india/comments/abd9dz...</td>\n",
       "      <td>askquestionsdude</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-01-01 00:45:17</td>\n",
       "      <td>0</td>\n",
       "      <td>/r/india/comments/abd9dz/allow_banks_to_hold_p...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[removed]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>abde0g</td>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>https://www.livemint.com/Politics/95rPBVTWmQqM...</td>\n",
       "      <td>askquestionsdude</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-01-01 01:00:15</td>\n",
       "      <td>3</td>\n",
       "      <td>/r/india/comments/abde0g/tamil_nadu_to_usher_i...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>abe9wz</td>\n",
       "      <td>Worst of the NPA crisis is over, says RBI report</td>\n",
       "      <td>https://www.livemint.com/Industry/LohId3yWEeQ1...</td>\n",
       "      <td>harddisc</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-01-01 02:56:12</td>\n",
       "      <td>0</td>\n",
       "      <td>/r/india/comments/abe9wz/worst_of_the_npa_cris...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aber36</td>\n",
       "      <td>Ravi Shastri's comments about O'Keefe</td>\n",
       "      <td>https://www.reddit.com/r/india/comments/aber36...</td>\n",
       "      <td>trowaaaay</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-01-01 04:03:40</td>\n",
       "      <td>3</td>\n",
       "      <td>/r/india/comments/aber36/ravi_shastris_comment...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>Kerry O' Keefe joked that Mayank Agarwal \"appa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>abervj</td>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>https://i.redd.it/s2gh1xg4kq721.jpg</td>\n",
       "      <td>Daiguren_Hyorinmaru_</td>\n",
       "      <td>1</td>\n",
       "      <td>2019-01-01 04:07:10</td>\n",
       "      <td>52</td>\n",
       "      <td>/r/india/comments/abervj/a_picture_i_clicked_a...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id                                              title  \\\n",
       "0  abd9dz  Allow banks to hold passports of loan-takers, ...   \n",
       "1  abde0g  Tamil Nadu to usher in New Year on green note ...   \n",
       "2  abe9wz   Worst of the NPA crisis is over, says RBI report   \n",
       "3  aber36              Ravi Shastri's comments about O'Keefe   \n",
       "4  abervj  A picture I clicked at the pillar rocks in Kod...   \n",
       "\n",
       "                                                 url                author  \\\n",
       "0  https://www.reddit.com/r/india/comments/abd9dz...      askquestionsdude   \n",
       "1  https://www.livemint.com/Politics/95rPBVTWmQqM...      askquestionsdude   \n",
       "2  https://www.livemint.com/Industry/LohId3yWEeQ1...              harddisc   \n",
       "3  https://www.reddit.com/r/india/comments/aber36...             trowaaaay   \n",
       "4                https://i.redd.it/s2gh1xg4kq721.jpg  Daiguren_Hyorinmaru_   \n",
       "\n",
       "   score         publish_date  num_comment  \\\n",
       "0      1  2019-01-01 00:45:17            0   \n",
       "1      1  2019-01-01 01:00:15            3   \n",
       "2      1  2019-01-01 02:56:12            0   \n",
       "3      1  2019-01-01 04:03:40            3   \n",
       "4      1  2019-01-01 04:07:10           52   \n",
       "\n",
       "                                           permalink             flair  \\\n",
       "0  /r/india/comments/abd9dz/allow_banks_to_hold_p...  Business/Finance   \n",
       "1  /r/india/comments/abde0g/tamil_nadu_to_usher_i...    Policy/Economy   \n",
       "2  /r/india/comments/abe9wz/worst_of_the_npa_cris...    Policy/Economy   \n",
       "3  /r/india/comments/aber36/ravi_shastris_comment...            Sports   \n",
       "4  /r/india/comments/abervj/a_picture_i_clicked_a...       Photography   \n",
       "\n",
       "                                            selftext  \n",
       "0                                          [removed]  \n",
       "1                                                NaN  \n",
       "2                                                NaN  \n",
       "3  Kerry O' Keefe joked that Mayank Agarwal \"appa...  \n",
       "4                                                NaN  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merging Title and Self-Text as one\n",
    "As the self-text is the extendent description of the title, the self-text and the tiles are mereged together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan 15673\n",
      "[deleted] 55\n",
      "[removed] 2019\n"
     ]
    }
   ],
   "source": [
    "data['selftext'] = data['selftext'].astype(str)\n",
    "empties = ['nan', '[deleted]', '[removed]']\n",
    "for empty in empties:\n",
    "  print(empty, len(data[data['selftext'] == empty]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "replacting nan, [deleted], [removed] and replacing with empty character \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan 0\n",
      "[deleted] 0\n",
      "[removed] 0\n"
     ]
    }
   ],
   "source": [
    "data['selftext'] = data['selftext'].apply(lambda x: \"\" if x in empties else x)\n",
    "for empty in empties:\n",
    "  print(empty, len(data[data['selftext'] == empty]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below we merge the title and the selftext parts and drop the unnecessary columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan-takers, ...</td>\n",
       "      <td>Business/Finance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over, says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri's comments about O'Keefe Kerry O'...</td>\n",
       "      <td>Sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title             flair\n",
       "0  Allow banks to hold passports of loan-takers, ...  Business/Finance\n",
       "1  Tamil Nadu to usher in New Year on green note ...    Policy/Economy\n",
       "2  Worst of the NPA crisis is over, says RBI report     Policy/Economy\n",
       "3  Ravi Shastri's comments about O'Keefe Kerry O'...            Sports\n",
       "4  A picture I clicked at the pillar rocks in Kod...       Photography"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['title'] = data['title'] + \" \" + data['selftext']\n",
    "data = data.drop(['id', 'url', 'author', 'score', 'publish_date', 'num_comment', 'permalink', 'selftext'], axis = 1)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['title'] = data['title'].apply(lambda x : re.compile(r'https?:\\/\\/\\S+').sub(\" \", x) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing Punctuations\n",
    "Removing all the symbols and punctuation and multiple-continuous puntuations like \"!!!\", \"...\", etc.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['title'] = data['title'].apply(lambda x : re.sub(r\"[^a-zA-Z]+\", r\" \", x) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Expanding Contractions\n",
    "Contractions are those little literary shortcuts we take where instead of “Should have” we prefer “Should’ve” or where “Do not” quickly becomes “Don’t”. We are going to add a new column to our dataframe called “no_contract” and apply a lambda function to the “rating_description” field which will expand any contractions. Be aware of the fact the expanded contractions will be effectively tokenized together. In other words, “I’ve” = “I have” instead of “I”, “have”.\n",
    "\n",
    "We ultimately would want the expanded contractions to be tokenized separately into “I”, “have”, therefore, let’s convert the lists under the “no_contract” column back into strings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['no_contract'] = data['title'].apply(lambda x :[contractions.fix(word) for word in x.split()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "      <th>no_contract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20743</th>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20744</th>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20745</th>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20746</th>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20747</th>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20748 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title             flair  \\\n",
       "0      Allow banks to hold passports of loan takers M...  Business/Finance   \n",
       "1      Tamil Nadu to usher in New Year on green note ...    Policy/Economy   \n",
       "2       Worst of the NPA crisis is over says RBI report     Policy/Economy   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...            Sports   \n",
       "4      A picture I clicked at the pillar rocks in Kod...       Photography   \n",
       "...                                                  ...               ...   \n",
       "20743  India leaves behind local transmission phase m...       Coronavirus   \n",
       "20744  Here s my side of the story as a small busines...    Policy/Economy   \n",
       "20745  Indian astronauts training on hold in Russia d...       Coronavirus   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...       Coronavirus   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...       Coronavirus   \n",
       "\n",
       "                                             no_contract  \n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...  \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...  \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...  \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...  \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...  \n",
       "...                                                  ...  \n",
       "20743  [India, leaves, behind, local, transmission, p...  \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...  \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...  \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...  \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...  \n",
       "\n",
       "[20748 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['title_description'] = [\" \".join(map(str, l)) for l in data['no_contract']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "      <th>no_contract</th>\n",
       "      <th>title_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20743</th>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20744</th>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20745</th>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20746</th>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20747</th>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20748 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title             flair  \\\n",
       "0      Allow banks to hold passports of loan takers M...  Business/Finance   \n",
       "1      Tamil Nadu to usher in New Year on green note ...    Policy/Economy   \n",
       "2       Worst of the NPA crisis is over says RBI report     Policy/Economy   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...            Sports   \n",
       "4      A picture I clicked at the pillar rocks in Kod...       Photography   \n",
       "...                                                  ...               ...   \n",
       "20743  India leaves behind local transmission phase m...       Coronavirus   \n",
       "20744  Here s my side of the story as a small busines...    Policy/Economy   \n",
       "20745  Indian astronauts training on hold in Russia d...       Coronavirus   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...       Coronavirus   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...       Coronavirus   \n",
       "\n",
       "                                             no_contract  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                       title_description  \n",
       "0      Allow banks to hold passports of loan takers M...  \n",
       "1      Tamil Nadu to usher in New Year on green note ...  \n",
       "2        Worst of the NPA crisis is over says RBI report  \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...  \n",
       "4      A picture I clicked at the pillar rocks in Kod...  \n",
       "...                                                  ...  \n",
       "20743  India leaves behind local transmission phase m...  \n",
       "20744  Here s my side of the story as a small busines...  \n",
       "20745  Indian astronauts training on hold in Russia d...  \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...  \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...  \n",
       "\n",
       "[20748 rows x 4 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization\n",
    "Now that we have converted the lists back into strings we can apply proper tokenization in order to split each individual word into a token. We will apply NLTK.word_tokenize() function to the “rating_description_list” column and create a new column named “tokenized”."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "      <th>no_contract</th>\n",
       "      <th>title_description</th>\n",
       "      <th>tokenized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20743</th>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20744</th>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20745</th>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20746</th>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20747</th>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20748 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title             flair  \\\n",
       "0      Allow banks to hold passports of loan takers M...  Business/Finance   \n",
       "1      Tamil Nadu to usher in New Year on green note ...    Policy/Economy   \n",
       "2       Worst of the NPA crisis is over says RBI report     Policy/Economy   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...            Sports   \n",
       "4      A picture I clicked at the pillar rocks in Kod...       Photography   \n",
       "...                                                  ...               ...   \n",
       "20743  India leaves behind local transmission phase m...       Coronavirus   \n",
       "20744  Here s my side of the story as a small busines...    Policy/Economy   \n",
       "20745  Indian astronauts training on hold in Russia d...       Coronavirus   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...       Coronavirus   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...       Coronavirus   \n",
       "\n",
       "                                             no_contract  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                       title_description  \\\n",
       "0      Allow banks to hold passports of loan takers M...   \n",
       "1      Tamil Nadu to usher in New Year on green note ...   \n",
       "2        Worst of the NPA crisis is over says RBI report   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...   \n",
       "4      A picture I clicked at the pillar rocks in Kod...   \n",
       "...                                                  ...   \n",
       "20743  India leaves behind local transmission phase m...   \n",
       "20744  Here s my side of the story as a small busines...   \n",
       "20745  Indian astronauts training on hold in Russia d...   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...   \n",
       "\n",
       "                                               tokenized  \n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...  \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...  \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...  \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...  \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...  \n",
       "...                                                  ...  \n",
       "20743  [India, leaves, behind, local, transmission, p...  \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...  \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...  \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...  \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...  \n",
       "\n",
       "[20748 rows x 5 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['tokenized'] = data['title_description'].apply(word_tokenize)\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting all Characters to Lowercase\n",
    "Transforming all words to lowercase is also a very common pre-processing step. In this case, we will once again append a new column named “lower” to the dataframe which will transform all the tokenized words into lowercase. However, because we have to iterate over multiple words we will use a simple for-loop within a lambda function to apply the “lower” function to each word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "      <th>no_contract</th>\n",
       "      <th>title_description</th>\n",
       "      <th>tokenized</th>\n",
       "      <th>lower</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>[tamil, nadu, to, usher, in, new, year, on, gr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, of, the, npa, crisis, is, over, says, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>[ravi, shastri, s, comments, about, o, keefe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[a, picture, i, clicked, at, the, pillar, rock...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20743</th>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20744</th>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20745</th>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>[indian, astronauts, training, on, hold, in, r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20746</th>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20747</th>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20748 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title             flair  \\\n",
       "0      Allow banks to hold passports of loan takers M...  Business/Finance   \n",
       "1      Tamil Nadu to usher in New Year on green note ...    Policy/Economy   \n",
       "2       Worst of the NPA crisis is over says RBI report     Policy/Economy   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...            Sports   \n",
       "4      A picture I clicked at the pillar rocks in Kod...       Photography   \n",
       "...                                                  ...               ...   \n",
       "20743  India leaves behind local transmission phase m...       Coronavirus   \n",
       "20744  Here s my side of the story as a small busines...    Policy/Economy   \n",
       "20745  Indian astronauts training on hold in Russia d...       Coronavirus   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...       Coronavirus   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...       Coronavirus   \n",
       "\n",
       "                                             no_contract  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                       title_description  \\\n",
       "0      Allow banks to hold passports of loan takers M...   \n",
       "1      Tamil Nadu to usher in New Year on green note ...   \n",
       "2        Worst of the NPA crisis is over says RBI report   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...   \n",
       "4      A picture I clicked at the pillar rocks in Kod...   \n",
       "...                                                  ...   \n",
       "20743  India leaves behind local transmission phase m...   \n",
       "20744  Here s my side of the story as a small busines...   \n",
       "20745  Indian astronauts training on hold in Russia d...   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...   \n",
       "\n",
       "                                               tokenized  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                                   lower  \n",
       "0      [allow, banks, to, hold, passports, of, loan, ...  \n",
       "1      [tamil, nadu, to, usher, in, new, year, on, gr...  \n",
       "2      [worst, of, the, npa, crisis, is, over, says, ...  \n",
       "3      [ravi, shastri, s, comments, about, o, keefe, ...  \n",
       "4      [a, picture, i, clicked, at, the, pillar, rock...  \n",
       "...                                                  ...  \n",
       "20743  [india, leaves, behind, local, transmission, p...  \n",
       "20744  [here, s, my, side, of, the, story, as, a, sma...  \n",
       "20745  [indian, astronauts, training, on, hold, in, r...  \n",
       "20746  [covid, government, threatens, kashmir, doctor...  \n",
       "20747  [coronavirus, cases, doubling, every, days, se...  \n",
       "\n",
       "[20748 rows x 6 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['lower'] = data['tokenized'].apply(lambda x: [word.lower() for word in x])\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing Stopwords\n",
    "Stopwords are typically useless words and do not add much meaning to a sentence. In the English language common stopwords include “you, he, she, in, a, has, are, etc.”. First, we need to import the NLTK stopwords library and set our stopwords to “english”. We are going to add a new column “no_stopwords” which will remove the stopwords from the “no_punc” column since it has been tokenized, had been converted to lowercase and punctuation was removed. Once again a for-loop within a lambda function will iterate over the tokens in “no_punc” and only return the tokens which do no exist in our “stop_words” variable.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "      <th>no_contract</th>\n",
       "      <th>title_description</th>\n",
       "      <th>tokenized</th>\n",
       "      <th>lower</th>\n",
       "      <th>stopwords_removed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, hold, passports, loan, takers, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>[tamil, nadu, to, usher, in, new, year, on, gr...</td>\n",
       "      <td>[tamil, nadu, usher, new, year, green, note, b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, of, the, npa, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, npa, crisis, says, rbi, report]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>[ravi, shastri, s, comments, about, o, keefe, ...</td>\n",
       "      <td>[ravi, shastri, comments, keefe, kerry, keefe,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[a, picture, i, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[picture, clicked, pillar, rocks, kodaikanal, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20743</th>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20744</th>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[side, story, small, businessman]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20745</th>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>[indian, astronauts, training, on, hold, in, r...</td>\n",
       "      <td>[indian, astronauts, training, hold, russia, d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20746</th>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20747</th>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20748 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title             flair  \\\n",
       "0      Allow banks to hold passports of loan takers M...  Business/Finance   \n",
       "1      Tamil Nadu to usher in New Year on green note ...    Policy/Economy   \n",
       "2       Worst of the NPA crisis is over says RBI report     Policy/Economy   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...            Sports   \n",
       "4      A picture I clicked at the pillar rocks in Kod...       Photography   \n",
       "...                                                  ...               ...   \n",
       "20743  India leaves behind local transmission phase m...       Coronavirus   \n",
       "20744  Here s my side of the story as a small busines...    Policy/Economy   \n",
       "20745  Indian astronauts training on hold in Russia d...       Coronavirus   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...       Coronavirus   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...       Coronavirus   \n",
       "\n",
       "                                             no_contract  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                       title_description  \\\n",
       "0      Allow banks to hold passports of loan takers M...   \n",
       "1      Tamil Nadu to usher in New Year on green note ...   \n",
       "2        Worst of the NPA crisis is over says RBI report   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...   \n",
       "4      A picture I clicked at the pillar rocks in Kod...   \n",
       "...                                                  ...   \n",
       "20743  India leaves behind local transmission phase m...   \n",
       "20744  Here s my side of the story as a small busines...   \n",
       "20745  Indian astronauts training on hold in Russia d...   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...   \n",
       "\n",
       "                                               tokenized  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                                   lower  \\\n",
       "0      [allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [tamil, nadu, to, usher, in, new, year, on, gr...   \n",
       "2      [worst, of, the, npa, crisis, is, over, says, ...   \n",
       "3      [ravi, shastri, s, comments, about, o, keefe, ...   \n",
       "4      [a, picture, i, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [india, leaves, behind, local, transmission, p...   \n",
       "20744  [here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [indian, astronauts, training, on, hold, in, r...   \n",
       "20746  [covid, government, threatens, kashmir, doctor...   \n",
       "20747  [coronavirus, cases, doubling, every, days, se...   \n",
       "\n",
       "                                       stopwords_removed  \n",
       "0      [allow, banks, hold, passports, loan, takers, ...  \n",
       "1      [tamil, nadu, usher, new, year, green, note, b...  \n",
       "2                [worst, npa, crisis, says, rbi, report]  \n",
       "3      [ravi, shastri, comments, keefe, kerry, keefe,...  \n",
       "4      [picture, clicked, pillar, rocks, kodaikanal, ...  \n",
       "...                                                  ...  \n",
       "20743  [india, leaves, behind, local, transmission, p...  \n",
       "20744                  [side, story, small, businessman]  \n",
       "20745  [indian, astronauts, training, hold, russia, d...  \n",
       "20746  [covid, government, threatens, kashmir, doctor...  \n",
       "20747  [coronavirus, cases, doubling, every, days, se...  \n",
       "\n",
       "[20748 rows x 7 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words = set(stopwords.words('english') + ['amp']) # added 'amp' to remove ampersands\n",
    "data['stopwords_removed'] = data['lower'].apply(lambda x: [word for word in x if word not in stop_words])\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lemmatization\n",
    "The idea of stemming is to reduce different forms of word usage into its root word. For example, “drive”, “drove”, “driving”, “driven”, “driver” are derivatives of the word “drive” and very often researchers want to remove this variability from their corpus. Compared to lemmatization, stemming is certainly the less complicated method but it often does not produce a dictionary-specific morphological root of the word. In other words, stemming the word “pies” will often produce a root of “pi” whereas lemmatization will find the morphological root of “pie”.\n",
    "\n",
    "Instead of taking the easy way out with stemming, let’s apply lemmatization to our data as it requires some additional steps compared to stemming.\n",
    "\n",
    "First, we have to apply parts of speech tags, in other words, determine the part of speech (ie. noun, verb, adverb, etc.) for each word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "      <th>no_contract</th>\n",
       "      <th>title_description</th>\n",
       "      <th>tokenized</th>\n",
       "      <th>lower</th>\n",
       "      <th>stopwords_removed</th>\n",
       "      <th>pos_tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, hold, passports, loan, takers, ...</td>\n",
       "      <td>[(allow, JJ), (banks, NNS), (hold, VBP), (pass...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>[tamil, nadu, to, usher, in, new, year, on, gr...</td>\n",
       "      <td>[tamil, nadu, usher, new, year, green, note, b...</td>\n",
       "      <td>[(tamil, NN), (nadu, MD), (usher, RB), (new, J...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, of, the, npa, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, npa, crisis, says, rbi, report]</td>\n",
       "      <td>[(worst, JJS), (npa, NN), (crisis, NN), (says,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>[ravi, shastri, s, comments, about, o, keefe, ...</td>\n",
       "      <td>[ravi, shastri, comments, keefe, kerry, keefe,...</td>\n",
       "      <td>[(ravi, NN), (shastri, NN), (comments, NNS), (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[a, picture, i, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[picture, clicked, pillar, rocks, kodaikanal, ...</td>\n",
       "      <td>[(picture, NN), (clicked, VBD), (pillar, JJ), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20743</th>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[(india, NN), (leaves, VBZ), (behind, IN), (lo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20744</th>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[side, story, small, businessman]</td>\n",
       "      <td>[(side, JJ), (story, NN), (small, JJ), (busine...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20745</th>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>[indian, astronauts, training, on, hold, in, r...</td>\n",
       "      <td>[indian, astronauts, training, hold, russia, d...</td>\n",
       "      <td>[(indian, JJ), (astronauts, NNS), (training, V...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20746</th>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "      <td>[(covid, JJ), (government, NN), (threatens, VB...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20747</th>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "      <td>[(coronavirus, NN), (cases, NNS), (doubling, V...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20748 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title             flair  \\\n",
       "0      Allow banks to hold passports of loan takers M...  Business/Finance   \n",
       "1      Tamil Nadu to usher in New Year on green note ...    Policy/Economy   \n",
       "2       Worst of the NPA crisis is over says RBI report     Policy/Economy   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...            Sports   \n",
       "4      A picture I clicked at the pillar rocks in Kod...       Photography   \n",
       "...                                                  ...               ...   \n",
       "20743  India leaves behind local transmission phase m...       Coronavirus   \n",
       "20744  Here s my side of the story as a small busines...    Policy/Economy   \n",
       "20745  Indian astronauts training on hold in Russia d...       Coronavirus   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...       Coronavirus   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...       Coronavirus   \n",
       "\n",
       "                                             no_contract  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                       title_description  \\\n",
       "0      Allow banks to hold passports of loan takers M...   \n",
       "1      Tamil Nadu to usher in New Year on green note ...   \n",
       "2        Worst of the NPA crisis is over says RBI report   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...   \n",
       "4      A picture I clicked at the pillar rocks in Kod...   \n",
       "...                                                  ...   \n",
       "20743  India leaves behind local transmission phase m...   \n",
       "20744  Here s my side of the story as a small busines...   \n",
       "20745  Indian astronauts training on hold in Russia d...   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...   \n",
       "\n",
       "                                               tokenized  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                                   lower  \\\n",
       "0      [allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [tamil, nadu, to, usher, in, new, year, on, gr...   \n",
       "2      [worst, of, the, npa, crisis, is, over, says, ...   \n",
       "3      [ravi, shastri, s, comments, about, o, keefe, ...   \n",
       "4      [a, picture, i, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [india, leaves, behind, local, transmission, p...   \n",
       "20744  [here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [indian, astronauts, training, on, hold, in, r...   \n",
       "20746  [covid, government, threatens, kashmir, doctor...   \n",
       "20747  [coronavirus, cases, doubling, every, days, se...   \n",
       "\n",
       "                                       stopwords_removed  \\\n",
       "0      [allow, banks, hold, passports, loan, takers, ...   \n",
       "1      [tamil, nadu, usher, new, year, green, note, b...   \n",
       "2                [worst, npa, crisis, says, rbi, report]   \n",
       "3      [ravi, shastri, comments, keefe, kerry, keefe,...   \n",
       "4      [picture, clicked, pillar, rocks, kodaikanal, ...   \n",
       "...                                                  ...   \n",
       "20743  [india, leaves, behind, local, transmission, p...   \n",
       "20744                  [side, story, small, businessman]   \n",
       "20745  [indian, astronauts, training, hold, russia, d...   \n",
       "20746  [covid, government, threatens, kashmir, doctor...   \n",
       "20747  [coronavirus, cases, doubling, every, days, se...   \n",
       "\n",
       "                                                pos_tags  \n",
       "0      [(allow, JJ), (banks, NNS), (hold, VBP), (pass...  \n",
       "1      [(tamil, NN), (nadu, MD), (usher, RB), (new, J...  \n",
       "2      [(worst, JJS), (npa, NN), (crisis, NN), (says,...  \n",
       "3      [(ravi, NN), (shastri, NN), (comments, NNS), (...  \n",
       "4      [(picture, NN), (clicked, VBD), (pillar, JJ), ...  \n",
       "...                                                  ...  \n",
       "20743  [(india, NN), (leaves, VBZ), (behind, IN), (lo...  \n",
       "20744  [(side, JJ), (story, NN), (small, JJ), (busine...  \n",
       "20745  [(indian, JJ), (astronauts, NNS), (training, V...  \n",
       "20746  [(covid, JJ), (government, NN), (threatens, VB...  \n",
       "20747  [(coronavirus, NN), (cases, NNS), (doubling, V...  \n",
       "\n",
       "[20748 rows x 8 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['pos_tags'] = data['stopwords_removed'].apply(nltk.tag.pos_tag)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_wordnet_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "      <th>no_contract</th>\n",
       "      <th>title_description</th>\n",
       "      <th>tokenized</th>\n",
       "      <th>lower</th>\n",
       "      <th>stopwords_removed</th>\n",
       "      <th>pos_tags</th>\n",
       "      <th>wordnet_pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, hold, passports, loan, takers, ...</td>\n",
       "      <td>[(allow, JJ), (banks, NNS), (hold, VBP), (pass...</td>\n",
       "      <td>[(allow, a), (banks, n), (hold, v), (passports...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>[tamil, nadu, to, usher, in, new, year, on, gr...</td>\n",
       "      <td>[tamil, nadu, usher, new, year, green, note, b...</td>\n",
       "      <td>[(tamil, NN), (nadu, MD), (usher, RB), (new, J...</td>\n",
       "      <td>[(tamil, n), (nadu, n), (usher, r), (new, a), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, of, the, npa, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, npa, crisis, says, rbi, report]</td>\n",
       "      <td>[(worst, JJS), (npa, NN), (crisis, NN), (says,...</td>\n",
       "      <td>[(worst, a), (npa, n), (crisis, n), (says, v),...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>[ravi, shastri, s, comments, about, o, keefe, ...</td>\n",
       "      <td>[ravi, shastri, comments, keefe, kerry, keefe,...</td>\n",
       "      <td>[(ravi, NN), (shastri, NN), (comments, NNS), (...</td>\n",
       "      <td>[(ravi, n), (shastri, n), (comments, n), (keef...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[a, picture, i, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[picture, clicked, pillar, rocks, kodaikanal, ...</td>\n",
       "      <td>[(picture, NN), (clicked, VBD), (pillar, JJ), ...</td>\n",
       "      <td>[(picture, n), (clicked, v), (pillar, a), (roc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20743</th>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[(india, NN), (leaves, VBZ), (behind, IN), (lo...</td>\n",
       "      <td>[(india, n), (leaves, v), (behind, n), (local,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20744</th>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[side, story, small, businessman]</td>\n",
       "      <td>[(side, JJ), (story, NN), (small, JJ), (busine...</td>\n",
       "      <td>[(side, a), (story, n), (small, a), (businessm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20745</th>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>[indian, astronauts, training, on, hold, in, r...</td>\n",
       "      <td>[indian, astronauts, training, hold, russia, d...</td>\n",
       "      <td>[(indian, JJ), (astronauts, NNS), (training, V...</td>\n",
       "      <td>[(indian, a), (astronauts, n), (training, v), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20746</th>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "      <td>[(covid, JJ), (government, NN), (threatens, VB...</td>\n",
       "      <td>[(covid, a), (government, n), (threatens, v), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20747</th>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "      <td>[(coronavirus, NN), (cases, NNS), (doubling, V...</td>\n",
       "      <td>[(coronavirus, n), (cases, n), (doubling, v), ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20748 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title             flair  \\\n",
       "0      Allow banks to hold passports of loan takers M...  Business/Finance   \n",
       "1      Tamil Nadu to usher in New Year on green note ...    Policy/Economy   \n",
       "2       Worst of the NPA crisis is over says RBI report     Policy/Economy   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...            Sports   \n",
       "4      A picture I clicked at the pillar rocks in Kod...       Photography   \n",
       "...                                                  ...               ...   \n",
       "20743  India leaves behind local transmission phase m...       Coronavirus   \n",
       "20744  Here s my side of the story as a small busines...    Policy/Economy   \n",
       "20745  Indian astronauts training on hold in Russia d...       Coronavirus   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...       Coronavirus   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...       Coronavirus   \n",
       "\n",
       "                                             no_contract  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                       title_description  \\\n",
       "0      Allow banks to hold passports of loan takers M...   \n",
       "1      Tamil Nadu to usher in New Year on green note ...   \n",
       "2        Worst of the NPA crisis is over says RBI report   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...   \n",
       "4      A picture I clicked at the pillar rocks in Kod...   \n",
       "...                                                  ...   \n",
       "20743  India leaves behind local transmission phase m...   \n",
       "20744  Here s my side of the story as a small busines...   \n",
       "20745  Indian astronauts training on hold in Russia d...   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...   \n",
       "\n",
       "                                               tokenized  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                                   lower  \\\n",
       "0      [allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [tamil, nadu, to, usher, in, new, year, on, gr...   \n",
       "2      [worst, of, the, npa, crisis, is, over, says, ...   \n",
       "3      [ravi, shastri, s, comments, about, o, keefe, ...   \n",
       "4      [a, picture, i, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [india, leaves, behind, local, transmission, p...   \n",
       "20744  [here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [indian, astronauts, training, on, hold, in, r...   \n",
       "20746  [covid, government, threatens, kashmir, doctor...   \n",
       "20747  [coronavirus, cases, doubling, every, days, se...   \n",
       "\n",
       "                                       stopwords_removed  \\\n",
       "0      [allow, banks, hold, passports, loan, takers, ...   \n",
       "1      [tamil, nadu, usher, new, year, green, note, b...   \n",
       "2                [worst, npa, crisis, says, rbi, report]   \n",
       "3      [ravi, shastri, comments, keefe, kerry, keefe,...   \n",
       "4      [picture, clicked, pillar, rocks, kodaikanal, ...   \n",
       "...                                                  ...   \n",
       "20743  [india, leaves, behind, local, transmission, p...   \n",
       "20744                  [side, story, small, businessman]   \n",
       "20745  [indian, astronauts, training, hold, russia, d...   \n",
       "20746  [covid, government, threatens, kashmir, doctor...   \n",
       "20747  [coronavirus, cases, doubling, every, days, se...   \n",
       "\n",
       "                                                pos_tags  \\\n",
       "0      [(allow, JJ), (banks, NNS), (hold, VBP), (pass...   \n",
       "1      [(tamil, NN), (nadu, MD), (usher, RB), (new, J...   \n",
       "2      [(worst, JJS), (npa, NN), (crisis, NN), (says,...   \n",
       "3      [(ravi, NN), (shastri, NN), (comments, NNS), (...   \n",
       "4      [(picture, NN), (clicked, VBD), (pillar, JJ), ...   \n",
       "...                                                  ...   \n",
       "20743  [(india, NN), (leaves, VBZ), (behind, IN), (lo...   \n",
       "20744  [(side, JJ), (story, NN), (small, JJ), (busine...   \n",
       "20745  [(indian, JJ), (astronauts, NNS), (training, V...   \n",
       "20746  [(covid, JJ), (government, NN), (threatens, VB...   \n",
       "20747  [(coronavirus, NN), (cases, NNS), (doubling, V...   \n",
       "\n",
       "                                             wordnet_pos  \n",
       "0      [(allow, a), (banks, n), (hold, v), (passports...  \n",
       "1      [(tamil, n), (nadu, n), (usher, r), (new, a), ...  \n",
       "2      [(worst, a), (npa, n), (crisis, n), (says, v),...  \n",
       "3      [(ravi, n), (shastri, n), (comments, n), (keef...  \n",
       "4      [(picture, n), (clicked, v), (pillar, a), (roc...  \n",
       "...                                                  ...  \n",
       "20743  [(india, n), (leaves, v), (behind, n), (local,...  \n",
       "20744  [(side, a), (story, n), (small, a), (businessm...  \n",
       "20745  [(indian, a), (astronauts, n), (training, v), ...  \n",
       "20746  [(covid, a), (government, n), (threatens, v), ...  \n",
       "20747  [(coronavirus, n), (cases, n), (doubling, v), ...  \n",
       "\n",
       "[20748 rows x 9 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['wordnet_pos'] = data['pos_tags'].apply(lambda x: [(word, get_wordnet_pos(pos_tag)) for (word, pos_tag) in x])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "      <th>no_contract</th>\n",
       "      <th>title_description</th>\n",
       "      <th>tokenized</th>\n",
       "      <th>lower</th>\n",
       "      <th>stopwords_removed</th>\n",
       "      <th>pos_tags</th>\n",
       "      <th>wordnet_pos</th>\n",
       "      <th>lemmatized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, hold, passports, loan, takers, ...</td>\n",
       "      <td>[(allow, JJ), (banks, NNS), (hold, VBP), (pass...</td>\n",
       "      <td>[(allow, a), (banks, n), (hold, v), (passports...</td>\n",
       "      <td>[allow, bank, hold, passport, loan, taker, mad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>[tamil, nadu, to, usher, in, new, year, on, gr...</td>\n",
       "      <td>[tamil, nadu, usher, new, year, green, note, b...</td>\n",
       "      <td>[(tamil, NN), (nadu, MD), (usher, RB), (new, J...</td>\n",
       "      <td>[(tamil, n), (nadu, n), (usher, r), (new, a), ...</td>\n",
       "      <td>[tamil, nadu, usher, new, year, green, note, b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, of, the, npa, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, npa, crisis, says, rbi, report]</td>\n",
       "      <td>[(worst, JJS), (npa, NN), (crisis, NN), (says,...</td>\n",
       "      <td>[(worst, a), (npa, n), (crisis, n), (says, v),...</td>\n",
       "      <td>[bad, npa, crisis, say, rbi, report]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>[ravi, shastri, s, comments, about, o, keefe, ...</td>\n",
       "      <td>[ravi, shastri, comments, keefe, kerry, keefe,...</td>\n",
       "      <td>[(ravi, NN), (shastri, NN), (comments, NNS), (...</td>\n",
       "      <td>[(ravi, n), (shastri, n), (comments, n), (keef...</td>\n",
       "      <td>[ravi, shastri, comment, keefe, kerry, keefe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[a, picture, i, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[picture, clicked, pillar, rocks, kodaikanal, ...</td>\n",
       "      <td>[(picture, NN), (clicked, VBD), (pillar, JJ), ...</td>\n",
       "      <td>[(picture, n), (clicked, v), (pillar, a), (roc...</td>\n",
       "      <td>[picture, click, pillar, rock, kodaikanal, rec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20743</th>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[(india, NN), (leaves, VBZ), (behind, IN), (lo...</td>\n",
       "      <td>[(india, n), (leaves, v), (behind, n), (local,...</td>\n",
       "      <td>[india, leave, behind, local, transmission, ph...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20744</th>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[side, story, small, businessman]</td>\n",
       "      <td>[(side, JJ), (story, NN), (small, JJ), (busine...</td>\n",
       "      <td>[(side, a), (story, n), (small, a), (businessm...</td>\n",
       "      <td>[side, story, small, businessman]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20745</th>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>[indian, astronauts, training, on, hold, in, r...</td>\n",
       "      <td>[indian, astronauts, training, hold, russia, d...</td>\n",
       "      <td>[(indian, JJ), (astronauts, NNS), (training, V...</td>\n",
       "      <td>[(indian, a), (astronauts, n), (training, v), ...</td>\n",
       "      <td>[indian, astronaut, train, hold, russia, due, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20746</th>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "      <td>[(covid, JJ), (government, NN), (threatens, VB...</td>\n",
       "      <td>[(covid, a), (government, n), (threatens, v), ...</td>\n",
       "      <td>[covid, government, threaten, kashmir, doctor,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20747</th>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "      <td>[(coronavirus, NN), (cases, NNS), (doubling, V...</td>\n",
       "      <td>[(coronavirus, n), (cases, n), (doubling, v), ...</td>\n",
       "      <td>[coronavirus, case, double, every, day, set, t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20748 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title             flair  \\\n",
       "0      Allow banks to hold passports of loan takers M...  Business/Finance   \n",
       "1      Tamil Nadu to usher in New Year on green note ...    Policy/Economy   \n",
       "2       Worst of the NPA crisis is over says RBI report     Policy/Economy   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...            Sports   \n",
       "4      A picture I clicked at the pillar rocks in Kod...       Photography   \n",
       "...                                                  ...               ...   \n",
       "20743  India leaves behind local transmission phase m...       Coronavirus   \n",
       "20744  Here s my side of the story as a small busines...    Policy/Economy   \n",
       "20745  Indian astronauts training on hold in Russia d...       Coronavirus   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...       Coronavirus   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...       Coronavirus   \n",
       "\n",
       "                                             no_contract  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                       title_description  \\\n",
       "0      Allow banks to hold passports of loan takers M...   \n",
       "1      Tamil Nadu to usher in New Year on green note ...   \n",
       "2        Worst of the NPA crisis is over says RBI report   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...   \n",
       "4      A picture I clicked at the pillar rocks in Kod...   \n",
       "...                                                  ...   \n",
       "20743  India leaves behind local transmission phase m...   \n",
       "20744  Here s my side of the story as a small busines...   \n",
       "20745  Indian astronauts training on hold in Russia d...   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...   \n",
       "\n",
       "                                               tokenized  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                                   lower  \\\n",
       "0      [allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [tamil, nadu, to, usher, in, new, year, on, gr...   \n",
       "2      [worst, of, the, npa, crisis, is, over, says, ...   \n",
       "3      [ravi, shastri, s, comments, about, o, keefe, ...   \n",
       "4      [a, picture, i, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [india, leaves, behind, local, transmission, p...   \n",
       "20744  [here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [indian, astronauts, training, on, hold, in, r...   \n",
       "20746  [covid, government, threatens, kashmir, doctor...   \n",
       "20747  [coronavirus, cases, doubling, every, days, se...   \n",
       "\n",
       "                                       stopwords_removed  \\\n",
       "0      [allow, banks, hold, passports, loan, takers, ...   \n",
       "1      [tamil, nadu, usher, new, year, green, note, b...   \n",
       "2                [worst, npa, crisis, says, rbi, report]   \n",
       "3      [ravi, shastri, comments, keefe, kerry, keefe,...   \n",
       "4      [picture, clicked, pillar, rocks, kodaikanal, ...   \n",
       "...                                                  ...   \n",
       "20743  [india, leaves, behind, local, transmission, p...   \n",
       "20744                  [side, story, small, businessman]   \n",
       "20745  [indian, astronauts, training, hold, russia, d...   \n",
       "20746  [covid, government, threatens, kashmir, doctor...   \n",
       "20747  [coronavirus, cases, doubling, every, days, se...   \n",
       "\n",
       "                                                pos_tags  \\\n",
       "0      [(allow, JJ), (banks, NNS), (hold, VBP), (pass...   \n",
       "1      [(tamil, NN), (nadu, MD), (usher, RB), (new, J...   \n",
       "2      [(worst, JJS), (npa, NN), (crisis, NN), (says,...   \n",
       "3      [(ravi, NN), (shastri, NN), (comments, NNS), (...   \n",
       "4      [(picture, NN), (clicked, VBD), (pillar, JJ), ...   \n",
       "...                                                  ...   \n",
       "20743  [(india, NN), (leaves, VBZ), (behind, IN), (lo...   \n",
       "20744  [(side, JJ), (story, NN), (small, JJ), (busine...   \n",
       "20745  [(indian, JJ), (astronauts, NNS), (training, V...   \n",
       "20746  [(covid, JJ), (government, NN), (threatens, VB...   \n",
       "20747  [(coronavirus, NN), (cases, NNS), (doubling, V...   \n",
       "\n",
       "                                             wordnet_pos  \\\n",
       "0      [(allow, a), (banks, n), (hold, v), (passports...   \n",
       "1      [(tamil, n), (nadu, n), (usher, r), (new, a), ...   \n",
       "2      [(worst, a), (npa, n), (crisis, n), (says, v),...   \n",
       "3      [(ravi, n), (shastri, n), (comments, n), (keef...   \n",
       "4      [(picture, n), (clicked, v), (pillar, a), (roc...   \n",
       "...                                                  ...   \n",
       "20743  [(india, n), (leaves, v), (behind, n), (local,...   \n",
       "20744  [(side, a), (story, n), (small, a), (businessm...   \n",
       "20745  [(indian, a), (astronauts, n), (training, v), ...   \n",
       "20746  [(covid, a), (government, n), (threatens, v), ...   \n",
       "20747  [(coronavirus, n), (cases, n), (doubling, v), ...   \n",
       "\n",
       "                                              lemmatized  \n",
       "0      [allow, bank, hold, passport, loan, taker, mad...  \n",
       "1      [tamil, nadu, usher, new, year, green, note, b...  \n",
       "2                   [bad, npa, crisis, say, rbi, report]  \n",
       "3      [ravi, shastri, comment, keefe, kerry, keefe, ...  \n",
       "4      [picture, click, pillar, rock, kodaikanal, rec...  \n",
       "...                                                  ...  \n",
       "20743  [india, leave, behind, local, transmission, ph...  \n",
       "20744                  [side, story, small, businessman]  \n",
       "20745  [indian, astronaut, train, hold, russia, due, ...  \n",
       "20746  [covid, government, threaten, kashmir, doctor,...  \n",
       "20747  [coronavirus, case, double, every, day, set, t...  \n",
       "\n",
       "[20748 rows x 10 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wnl  = WordNetLemmatizer()\n",
    "data['lemmatized'] = data['wordnet_pos'].apply(lambda x: [wnl.lemmatize(word, tag) for word, tag in x])\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Removing Small Words\n",
    "Remove all words with length < 2 from the data, some redundant and unrecognized small words may not be in the stopword vocabulary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>flair</th>\n",
       "      <th>no_contract</th>\n",
       "      <th>title_description</th>\n",
       "      <th>tokenized</th>\n",
       "      <th>lower</th>\n",
       "      <th>stopwords_removed</th>\n",
       "      <th>pos_tags</th>\n",
       "      <th>wordnet_pos</th>\n",
       "      <th>lemmatized</th>\n",
       "      <th>small_words_rem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>Business/Finance</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>Allow banks to hold passports of loan takers M...</td>\n",
       "      <td>[Allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, to, hold, passports, of, loan, ...</td>\n",
       "      <td>[allow, banks, hold, passports, loan, takers, ...</td>\n",
       "      <td>[(allow, JJ), (banks, NNS), (hold, VBP), (pass...</td>\n",
       "      <td>[(allow, a), (banks, n), (hold, v), (passports...</td>\n",
       "      <td>[allow, bank, hold, passport, loan, taker, mad...</td>\n",
       "      <td>[allow, bank, hold, passport, loan, taker, mad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>Tamil Nadu to usher in New Year on green note ...</td>\n",
       "      <td>[Tamil, Nadu, to, usher, in, New, Year, on, gr...</td>\n",
       "      <td>[tamil, nadu, to, usher, in, new, year, on, gr...</td>\n",
       "      <td>[tamil, nadu, usher, new, year, green, note, b...</td>\n",
       "      <td>[(tamil, NN), (nadu, MD), (usher, RB), (new, J...</td>\n",
       "      <td>[(tamil, n), (nadu, n), (usher, r), (new, a), ...</td>\n",
       "      <td>[tamil, nadu, usher, new, year, green, note, b...</td>\n",
       "      <td>[tamil, nadu, usher, new, year, green, note, b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>Worst of the NPA crisis is over says RBI report</td>\n",
       "      <td>[Worst, of, the, NPA, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, of, the, npa, crisis, is, over, says, ...</td>\n",
       "      <td>[worst, npa, crisis, says, rbi, report]</td>\n",
       "      <td>[(worst, JJS), (npa, NN), (crisis, NN), (says,...</td>\n",
       "      <td>[(worst, a), (npa, n), (crisis, n), (says, v),...</td>\n",
       "      <td>[bad, npa, crisis, say, rbi, report]</td>\n",
       "      <td>[bad, npa, crisis, say, rbi, report]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>Sports</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>Ravi Shastri s comments about O Keefe Kerry O ...</td>\n",
       "      <td>[Ravi, Shastri, s, comments, about, O, Keefe, ...</td>\n",
       "      <td>[ravi, shastri, s, comments, about, o, keefe, ...</td>\n",
       "      <td>[ravi, shastri, comments, keefe, kerry, keefe,...</td>\n",
       "      <td>[(ravi, NN), (shastri, NN), (comments, NNS), (...</td>\n",
       "      <td>[(ravi, n), (shastri, n), (comments, n), (keef...</td>\n",
       "      <td>[ravi, shastri, comment, keefe, kerry, keefe, ...</td>\n",
       "      <td>[ravi, shastri, comment, keefe, kerry, keefe, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>Photography</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>A picture I clicked at the pillar rocks in Kod...</td>\n",
       "      <td>[A, picture, I, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[a, picture, i, clicked, at, the, pillar, rock...</td>\n",
       "      <td>[picture, clicked, pillar, rocks, kodaikanal, ...</td>\n",
       "      <td>[(picture, NN), (clicked, VBD), (pillar, JJ), ...</td>\n",
       "      <td>[(picture, n), (clicked, v), (pillar, a), (roc...</td>\n",
       "      <td>[picture, click, pillar, rock, kodaikanal, rec...</td>\n",
       "      <td>[picture, click, pillar, rock, kodaikanal, rec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20743</th>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>India leaves behind local transmission phase m...</td>\n",
       "      <td>[India, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[india, leaves, behind, local, transmission, p...</td>\n",
       "      <td>[(india, NN), (leaves, VBZ), (behind, IN), (lo...</td>\n",
       "      <td>[(india, n), (leaves, v), (behind, n), (local,...</td>\n",
       "      <td>[india, leave, behind, local, transmission, ph...</td>\n",
       "      <td>[india, leave, behind, local, transmission, ph...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20744</th>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>Policy/Economy</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>Here s my side of the story as a small busines...</td>\n",
       "      <td>[Here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[here, s, my, side, of, the, story, as, a, sma...</td>\n",
       "      <td>[side, story, small, businessman]</td>\n",
       "      <td>[(side, JJ), (story, NN), (small, JJ), (busine...</td>\n",
       "      <td>[(side, a), (story, n), (small, a), (businessm...</td>\n",
       "      <td>[side, story, small, businessman]</td>\n",
       "      <td>[side, story, small, businessman]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20745</th>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>Indian astronauts training on hold in Russia d...</td>\n",
       "      <td>[Indian, astronauts, training, on, hold, in, R...</td>\n",
       "      <td>[indian, astronauts, training, on, hold, in, r...</td>\n",
       "      <td>[indian, astronauts, training, hold, russia, d...</td>\n",
       "      <td>[(indian, JJ), (astronauts, NNS), (training, V...</td>\n",
       "      <td>[(indian, a), (astronauts, n), (training, v), ...</td>\n",
       "      <td>[indian, astronaut, train, hold, russia, due, ...</td>\n",
       "      <td>[indian, astronaut, train, hold, russia, due, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20746</th>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>COVID Government Threatens Kashmir Doctors Wit...</td>\n",
       "      <td>[COVID, Government, Threatens, Kashmir, Doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "      <td>[covid, government, threatens, kashmir, doctor...</td>\n",
       "      <td>[(covid, JJ), (government, NN), (threatens, VB...</td>\n",
       "      <td>[(covid, a), (government, n), (threatens, v), ...</td>\n",
       "      <td>[covid, government, threaten, kashmir, doctor,...</td>\n",
       "      <td>[covid, government, threaten, kashmir, doctor,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20747</th>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>Coronavirus Cases Doubling Every Days Set To T...</td>\n",
       "      <td>[Coronavirus, Cases, Doubling, Every, Days, Se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "      <td>[coronavirus, cases, doubling, every, days, se...</td>\n",
       "      <td>[(coronavirus, NN), (cases, NNS), (doubling, V...</td>\n",
       "      <td>[(coronavirus, n), (cases, n), (doubling, v), ...</td>\n",
       "      <td>[coronavirus, case, double, every, day, set, t...</td>\n",
       "      <td>[coronavirus, case, double, every, day, set, t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20748 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title             flair  \\\n",
       "0      Allow banks to hold passports of loan takers M...  Business/Finance   \n",
       "1      Tamil Nadu to usher in New Year on green note ...    Policy/Economy   \n",
       "2       Worst of the NPA crisis is over says RBI report     Policy/Economy   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...            Sports   \n",
       "4      A picture I clicked at the pillar rocks in Kod...       Photography   \n",
       "...                                                  ...               ...   \n",
       "20743  India leaves behind local transmission phase m...       Coronavirus   \n",
       "20744  Here s my side of the story as a small busines...    Policy/Economy   \n",
       "20745  Indian astronauts training on hold in Russia d...       Coronavirus   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...       Coronavirus   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...       Coronavirus   \n",
       "\n",
       "                                             no_contract  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                       title_description  \\\n",
       "0      Allow banks to hold passports of loan takers M...   \n",
       "1      Tamil Nadu to usher in New Year on green note ...   \n",
       "2        Worst of the NPA crisis is over says RBI report   \n",
       "3      Ravi Shastri s comments about O Keefe Kerry O ...   \n",
       "4      A picture I clicked at the pillar rocks in Kod...   \n",
       "...                                                  ...   \n",
       "20743  India leaves behind local transmission phase m...   \n",
       "20744  Here s my side of the story as a small busines...   \n",
       "20745  Indian astronauts training on hold in Russia d...   \n",
       "20746  COVID Government Threatens Kashmir Doctors Wit...   \n",
       "20747  Coronavirus Cases Doubling Every Days Set To T...   \n",
       "\n",
       "                                               tokenized  \\\n",
       "0      [Allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [Tamil, Nadu, to, usher, in, New, Year, on, gr...   \n",
       "2      [Worst, of, the, NPA, crisis, is, over, says, ...   \n",
       "3      [Ravi, Shastri, s, comments, about, O, Keefe, ...   \n",
       "4      [A, picture, I, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [India, leaves, behind, local, transmission, p...   \n",
       "20744  [Here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [Indian, astronauts, training, on, hold, in, R...   \n",
       "20746  [COVID, Government, Threatens, Kashmir, Doctor...   \n",
       "20747  [Coronavirus, Cases, Doubling, Every, Days, Se...   \n",
       "\n",
       "                                                   lower  \\\n",
       "0      [allow, banks, to, hold, passports, of, loan, ...   \n",
       "1      [tamil, nadu, to, usher, in, new, year, on, gr...   \n",
       "2      [worst, of, the, npa, crisis, is, over, says, ...   \n",
       "3      [ravi, shastri, s, comments, about, o, keefe, ...   \n",
       "4      [a, picture, i, clicked, at, the, pillar, rock...   \n",
       "...                                                  ...   \n",
       "20743  [india, leaves, behind, local, transmission, p...   \n",
       "20744  [here, s, my, side, of, the, story, as, a, sma...   \n",
       "20745  [indian, astronauts, training, on, hold, in, r...   \n",
       "20746  [covid, government, threatens, kashmir, doctor...   \n",
       "20747  [coronavirus, cases, doubling, every, days, se...   \n",
       "\n",
       "                                       stopwords_removed  \\\n",
       "0      [allow, banks, hold, passports, loan, takers, ...   \n",
       "1      [tamil, nadu, usher, new, year, green, note, b...   \n",
       "2                [worst, npa, crisis, says, rbi, report]   \n",
       "3      [ravi, shastri, comments, keefe, kerry, keefe,...   \n",
       "4      [picture, clicked, pillar, rocks, kodaikanal, ...   \n",
       "...                                                  ...   \n",
       "20743  [india, leaves, behind, local, transmission, p...   \n",
       "20744                  [side, story, small, businessman]   \n",
       "20745  [indian, astronauts, training, hold, russia, d...   \n",
       "20746  [covid, government, threatens, kashmir, doctor...   \n",
       "20747  [coronavirus, cases, doubling, every, days, se...   \n",
       "\n",
       "                                                pos_tags  \\\n",
       "0      [(allow, JJ), (banks, NNS), (hold, VBP), (pass...   \n",
       "1      [(tamil, NN), (nadu, MD), (usher, RB), (new, J...   \n",
       "2      [(worst, JJS), (npa, NN), (crisis, NN), (says,...   \n",
       "3      [(ravi, NN), (shastri, NN), (comments, NNS), (...   \n",
       "4      [(picture, NN), (clicked, VBD), (pillar, JJ), ...   \n",
       "...                                                  ...   \n",
       "20743  [(india, NN), (leaves, VBZ), (behind, IN), (lo...   \n",
       "20744  [(side, JJ), (story, NN), (small, JJ), (busine...   \n",
       "20745  [(indian, JJ), (astronauts, NNS), (training, V...   \n",
       "20746  [(covid, JJ), (government, NN), (threatens, VB...   \n",
       "20747  [(coronavirus, NN), (cases, NNS), (doubling, V...   \n",
       "\n",
       "                                             wordnet_pos  \\\n",
       "0      [(allow, a), (banks, n), (hold, v), (passports...   \n",
       "1      [(tamil, n), (nadu, n), (usher, r), (new, a), ...   \n",
       "2      [(worst, a), (npa, n), (crisis, n), (says, v),...   \n",
       "3      [(ravi, n), (shastri, n), (comments, n), (keef...   \n",
       "4      [(picture, n), (clicked, v), (pillar, a), (roc...   \n",
       "...                                                  ...   \n",
       "20743  [(india, n), (leaves, v), (behind, n), (local,...   \n",
       "20744  [(side, a), (story, n), (small, a), (businessm...   \n",
       "20745  [(indian, a), (astronauts, n), (training, v), ...   \n",
       "20746  [(covid, a), (government, n), (threatens, v), ...   \n",
       "20747  [(coronavirus, n), (cases, n), (doubling, v), ...   \n",
       "\n",
       "                                              lemmatized  \\\n",
       "0      [allow, bank, hold, passport, loan, taker, mad...   \n",
       "1      [tamil, nadu, usher, new, year, green, note, b...   \n",
       "2                   [bad, npa, crisis, say, rbi, report]   \n",
       "3      [ravi, shastri, comment, keefe, kerry, keefe, ...   \n",
       "4      [picture, click, pillar, rock, kodaikanal, rec...   \n",
       "...                                                  ...   \n",
       "20743  [india, leave, behind, local, transmission, ph...   \n",
       "20744                  [side, story, small, businessman]   \n",
       "20745  [indian, astronaut, train, hold, russia, due, ...   \n",
       "20746  [covid, government, threaten, kashmir, doctor,...   \n",
       "20747  [coronavirus, case, double, every, day, set, t...   \n",
       "\n",
       "                                         small_words_rem  \n",
       "0      [allow, bank, hold, passport, loan, taker, mad...  \n",
       "1      [tamil, nadu, usher, new, year, green, note, b...  \n",
       "2                   [bad, npa, crisis, say, rbi, report]  \n",
       "3      [ravi, shastri, comment, keefe, kerry, keefe, ...  \n",
       "4      [picture, click, pillar, rock, kodaikanal, rec...  \n",
       "...                                                  ...  \n",
       "20743  [india, leave, behind, local, transmission, ph...  \n",
       "20744                  [side, story, small, businessman]  \n",
       "20745  [indian, astronaut, train, hold, russia, due, ...  \n",
       "20746  [covid, government, threaten, kashmir, doctor,...  \n",
       "20747  [coronavirus, case, double, every, day, set, t...  \n",
       "\n",
       "[20748 rows x 11 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['small_words_rem'] = data['lemmatized'].apply(lambda x: [t for t in x if len(t) > 2])\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving the preprocessed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_pickle(\"data_cleaned.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}