10_stop_words/stop_words.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KYoVrnewenmh"
   },
   "source": [
    "### Stop words tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "326"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "\n",
    "len(STOP_WORDS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We\n",
      "just\n",
      "our\n",
      "the\n",
      "part\n",
      "is\n"
     ]
    }
   ],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "doc = nlp(\"We just opened our wings, the flying part is coming soon\")\n",
    "\n",
    "for token in doc:\n",
    "    if token.is_stop:\n",
    "        print(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    doc = nlp(text)\n",
    "    \n",
    "    no_stop_words = [token.text for token in doc if not token.is_stop]\n",
    "    return \" \".join(no_stop_words)            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Musk wants time prepare trial'"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(\"Musk wants time to prepare for a trial over his\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'divine brother'"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(\"The other is not other but your divine brother\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KYoVrnewenmh"
   },
   "source": [
    "##### Remove stop words from pandas dataframe text column"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset is downloaded from: https://www.kaggle.com/datasets/jbencina/department-of-justice-20092018-press-releases\n",
    "It contains press releases of different court cases from depart of justice (DOJ). The releases contain information such as outcomes of criminal cases, notable actions taken against felons, or other updates about the current administration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13087, 6)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_json(\"doj_press.json\",lines=True)\n",
    "\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>contents</th>\n",
       "      <th>date</th>\n",
       "      <th>topics</th>\n",
       "      <th>components</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>Convicted Bomb Plotter Sentenced to 30 Years</td>\n",
       "      <td>PORTLAND, Oregon. – Mohamed Osman Mohamud, 23,...</td>\n",
       "      <td>2014-10-01T00:00:00-04:00</td>\n",
       "      <td>[]</td>\n",
       "      <td>[National Security Division (NSD)]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12-919</td>\n",
       "      <td>$1 Million in Restitution Payments Announced t...</td>\n",
       "      <td>WASHINGTON – North Carolina’s Waccamaw River...</td>\n",
       "      <td>2012-07-25T00:00:00-04:00</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11-1002</td>\n",
       "      <td>$1 Million Settlement Reached for Natural Reso...</td>\n",
       "      <td>BOSTON– A $1-million settlement has been...</td>\n",
       "      <td>2011-08-03T00:00:00-04:00</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10-015</td>\n",
       "      <td>10 Las Vegas Men Indicted \\r\\nfor Falsifying V...</td>\n",
       "      <td>WASHINGTON—A federal grand jury in Las Vegas...</td>\n",
       "      <td>2010-01-08T00:00:00-05:00</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18-898</td>\n",
       "      <td>$100 Million Settlement Will Speed Cleanup Wor...</td>\n",
       "      <td>The U.S. Department of Justice, the U.S. Envir...</td>\n",
       "      <td>2018-07-09T00:00:00-04:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                              title  \\\n",
       "0     None       Convicted Bomb Plotter Sentenced to 30 Years   \n",
       "1  12-919   $1 Million in Restitution Payments Announced t...   \n",
       "2  11-1002  $1 Million Settlement Reached for Natural Reso...   \n",
       "3   10-015  10 Las Vegas Men Indicted \\r\\nfor Falsifying V...   \n",
       "4   18-898  $100 Million Settlement Will Speed Cleanup Wor...   \n",
       "\n",
       "                                            contents  \\\n",
       "0  PORTLAND, Oregon. – Mohamed Osman Mohamud, 23,...   \n",
       "1    WASHINGTON – North Carolina’s Waccamaw River...   \n",
       "2        BOSTON– A $1-million settlement has been...   \n",
       "3    WASHINGTON—A federal grand jury in Las Vegas...   \n",
       "4  The U.S. Department of Justice, the U.S. Envir...   \n",
       "\n",
       "                        date         topics  \\\n",
       "0  2014-10-01T00:00:00-04:00             []   \n",
       "1  2012-07-25T00:00:00-04:00             []   \n",
       "2  2011-08-03T00:00:00-04:00             []   \n",
       "3  2010-01-08T00:00:00-05:00             []   \n",
       "4  2018-07-09T00:00:00-04:00  [Environment]   \n",
       "\n",
       "                                     components  \n",
       "0            [National Security Division (NSD)]  \n",
       "1  [Environment and Natural Resources Division]  \n",
       "2  [Environment and Natural Resources Division]  \n",
       "3  [Environment and Natural Resources Division]  \n",
       "4  [Environment and Natural Resources Division]  "
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filter out those rows that do not have any topics associated with the case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>contents</th>\n",
       "      <th>date</th>\n",
       "      <th>topics</th>\n",
       "      <th>components</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18-898</td>\n",
       "      <td>$100 Million Settlement Will Speed Cleanup Wor...</td>\n",
       "      <td>The U.S. Department of Justice, the U.S. Envir...</td>\n",
       "      <td>2018-07-09T00:00:00-04:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>14-1412</td>\n",
       "      <td>14 Indicted in Connection with New England Com...</td>\n",
       "      <td>A 131-count criminal indictment was unsealed t...</td>\n",
       "      <td>2014-12-17T00:00:00-05:00</td>\n",
       "      <td>[Consumer Protection]</td>\n",
       "      <td>[Civil Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>17-1419</td>\n",
       "      <td>2017 Southeast Regional Animal Cruelty Prosecu...</td>\n",
       "      <td>The United States Attorney’s Office for the Mi...</td>\n",
       "      <td>2017-12-14T00:00:00-05:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division, U...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>15-1562</td>\n",
       "      <td>21st Century Oncology to Pay $19.75 Million to...</td>\n",
       "      <td>21st Century Oncology LLC, has agreed to pay $...</td>\n",
       "      <td>2015-12-18T00:00:00-05:00</td>\n",
       "      <td>[False Claims Act, Health Care Fraud]</td>\n",
       "      <td>[Civil Division]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>17-1404</td>\n",
       "      <td>21st Century Oncology to Pay $26 Million to Se...</td>\n",
       "      <td>21st Century Oncology Inc. and certain of its ...</td>\n",
       "      <td>2017-12-12T00:00:00-05:00</td>\n",
       "      <td>[Health Care Fraud, False Claims Act]</td>\n",
       "      <td>[Civil Division, USAO - Florida, Middle]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                              title  \\\n",
       "4    18-898  $100 Million Settlement Will Speed Cleanup Wor...   \n",
       "7   14-1412  14 Indicted in Connection with New England Com...   \n",
       "19  17-1419  2017 Southeast Regional Animal Cruelty Prosecu...   \n",
       "22  15-1562  21st Century Oncology to Pay $19.75 Million to...   \n",
       "23  17-1404  21st Century Oncology to Pay $26 Million to Se...   \n",
       "\n",
       "                                             contents  \\\n",
       "4   The U.S. Department of Justice, the U.S. Envir...   \n",
       "7   A 131-count criminal indictment was unsealed t...   \n",
       "19  The United States Attorney’s Office for the Mi...   \n",
       "22  21st Century Oncology LLC, has agreed to pay $...   \n",
       "23  21st Century Oncology Inc. and certain of its ...   \n",
       "\n",
       "                         date                                 topics  \\\n",
       "4   2018-07-09T00:00:00-04:00                          [Environment]   \n",
       "7   2014-12-17T00:00:00-05:00                  [Consumer Protection]   \n",
       "19  2017-12-14T00:00:00-05:00                          [Environment]   \n",
       "22  2015-12-18T00:00:00-05:00  [False Claims Act, Health Care Fraud]   \n",
       "23  2017-12-12T00:00:00-05:00  [Health Care Fraud, False Claims Act]   \n",
       "\n",
       "                                           components  \n",
       "4        [Environment and Natural Resources Division]  \n",
       "7                                    [Civil Division]  \n",
       "19  [Environment and Natural Resources Division, U...  \n",
       "22                                   [Civil Division]  \n",
       "23           [Civil Division, USAO - Florida, Middle]  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[df[\"topics\"].str.len() != 0]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13087, 6)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 6)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df =df.head(100)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"contents_new\"] = df.contents.apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>contents</th>\n",
       "      <th>date</th>\n",
       "      <th>topics</th>\n",
       "      <th>components</th>\n",
       "      <th>contents_new</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18-898</td>\n",
       "      <td>$100 Million Settlement Will Speed Cleanup Wor...</td>\n",
       "      <td>The U.S. Department of Justice, the U.S. Envir...</td>\n",
       "      <td>2018-07-09T00:00:00-04:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division]</td>\n",
       "      <td>U.S. Department Justice , U.S. Environmental P...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>14-1412</td>\n",
       "      <td>14 Indicted in Connection with New England Com...</td>\n",
       "      <td>A 131-count criminal indictment was unsealed t...</td>\n",
       "      <td>2014-12-17T00:00:00-05:00</td>\n",
       "      <td>[Consumer Protection]</td>\n",
       "      <td>[Civil Division]</td>\n",
       "      <td>131 - count criminal indictment unsealed today...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>17-1419</td>\n",
       "      <td>2017 Southeast Regional Animal Cruelty Prosecu...</td>\n",
       "      <td>The United States Attorney’s Office for the Mi...</td>\n",
       "      <td>2017-12-14T00:00:00-05:00</td>\n",
       "      <td>[Environment]</td>\n",
       "      <td>[Environment and Natural Resources Division, U...</td>\n",
       "      <td>United States Attorney Office Middle District ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>15-1562</td>\n",
       "      <td>21st Century Oncology to Pay $19.75 Million to...</td>\n",
       "      <td>21st Century Oncology LLC, has agreed to pay $...</td>\n",
       "      <td>2015-12-18T00:00:00-05:00</td>\n",
       "      <td>[False Claims Act, Health Care Fraud]</td>\n",
       "      <td>[Civil Division]</td>\n",
       "      <td>21st Century Oncology LLC , agreed pay $ 19.75...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>17-1404</td>\n",
       "      <td>21st Century Oncology to Pay $26 Million to Se...</td>\n",
       "      <td>21st Century Oncology Inc. and certain of its ...</td>\n",
       "      <td>2017-12-12T00:00:00-05:00</td>\n",
       "      <td>[Health Care Fraud, False Claims Act]</td>\n",
       "      <td>[Civil Division, USAO - Florida, Middle]</td>\n",
       "      <td>21st Century Oncology Inc. certain subsidiarie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316</th>\n",
       "      <td>15-1359</td>\n",
       "      <td>Alaska Plastic Surgeon Convicted of Wire Fraud...</td>\n",
       "      <td>Doctor Hid Millions in Secret Accounts in Pana...</td>\n",
       "      <td>2015-11-04T00:00:00-05:00</td>\n",
       "      <td>[Tax]</td>\n",
       "      <td>[Tax Division]</td>\n",
       "      <td>Doctor Hid Millions Secret Accounts Panama Ala...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>16-396</td>\n",
       "      <td>Alaska Plastic Surgeon Sentenced to Prison for...</td>\n",
       "      <td>Defendant Concealed Bank Accounts in Panama an...</td>\n",
       "      <td>2016-04-04T00:00:00-04:00</td>\n",
       "      <td>[Tax]</td>\n",
       "      <td>[Tax Division]</td>\n",
       "      <td>Defendant Concealed Bank Accounts Panama Costa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321</th>\n",
       "      <td>17-736</td>\n",
       "      <td>Alaskan Commercial Fishing Couple Charged with...</td>\n",
       "      <td>An Alaskan couple was charged in federal court...</td>\n",
       "      <td>2017-07-26T00:00:00-04:00</td>\n",
       "      <td>[Tax]</td>\n",
       "      <td>[Tax Division, USAO - Alaska]</td>\n",
       "      <td>Alaskan couple charged federal court Juneau , ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322</th>\n",
       "      <td>18-717</td>\n",
       "      <td>Alaskan Husband And Wife Plead Guilty To Willf...</td>\n",
       "      <td>A husband and wife pleaded guilty yesterday to...</td>\n",
       "      <td>2018-06-01T00:00:00-04:00</td>\n",
       "      <td>[Tax]</td>\n",
       "      <td>[Tax Division]</td>\n",
       "      <td>husband wife pleaded guilty yesterday counts w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>324</th>\n",
       "      <td>16-1345</td>\n",
       "      <td>Alaskan Oncologist Indicted for Tax Evasion</td>\n",
       "      <td>A resident of Big Lake, Alaska was indicted on...</td>\n",
       "      <td>2016-11-17T00:00:00-05:00</td>\n",
       "      <td>[Tax]</td>\n",
       "      <td>[Tax Division]</td>\n",
       "      <td>resident Big Lake , Alaska indicted counts tax...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                              title  \\\n",
       "4     18-898  $100 Million Settlement Will Speed Cleanup Wor...   \n",
       "7    14-1412  14 Indicted in Connection with New England Com...   \n",
       "19   17-1419  2017 Southeast Regional Animal Cruelty Prosecu...   \n",
       "22   15-1562  21st Century Oncology to Pay $19.75 Million to...   \n",
       "23   17-1404  21st Century Oncology to Pay $26 Million to Se...   \n",
       "..       ...                                                ...   \n",
       "316  15-1359  Alaska Plastic Surgeon Convicted of Wire Fraud...   \n",
       "318   16-396  Alaska Plastic Surgeon Sentenced to Prison for...   \n",
       "321   17-736  Alaskan Commercial Fishing Couple Charged with...   \n",
       "322   18-717  Alaskan Husband And Wife Plead Guilty To Willf...   \n",
       "324  16-1345        Alaskan Oncologist Indicted for Tax Evasion   \n",
       "\n",
       "                                              contents  \\\n",
       "4    The U.S. Department of Justice, the U.S. Envir...   \n",
       "7    A 131-count criminal indictment was unsealed t...   \n",
       "19   The United States Attorney’s Office for the Mi...   \n",
       "22   21st Century Oncology LLC, has agreed to pay $...   \n",
       "23   21st Century Oncology Inc. and certain of its ...   \n",
       "..                                                 ...   \n",
       "316  Doctor Hid Millions in Secret Accounts in Pana...   \n",
       "318  Defendant Concealed Bank Accounts in Panama an...   \n",
       "321  An Alaskan couple was charged in federal court...   \n",
       "322  A husband and wife pleaded guilty yesterday to...   \n",
       "324  A resident of Big Lake, Alaska was indicted on...   \n",
       "\n",
       "                          date                                 topics  \\\n",
       "4    2018-07-09T00:00:00-04:00                          [Environment]   \n",
       "7    2014-12-17T00:00:00-05:00                  [Consumer Protection]   \n",
       "19   2017-12-14T00:00:00-05:00                          [Environment]   \n",
       "22   2015-12-18T00:00:00-05:00  [False Claims Act, Health Care Fraud]   \n",
       "23   2017-12-12T00:00:00-05:00  [Health Care Fraud, False Claims Act]   \n",
       "..                         ...                                    ...   \n",
       "316  2015-11-04T00:00:00-05:00                                  [Tax]   \n",
       "318  2016-04-04T00:00:00-04:00                                  [Tax]   \n",
       "321  2017-07-26T00:00:00-04:00                                  [Tax]   \n",
       "322  2018-06-01T00:00:00-04:00                                  [Tax]   \n",
       "324  2016-11-17T00:00:00-05:00                                  [Tax]   \n",
       "\n",
       "                                            components  \\\n",
       "4         [Environment and Natural Resources Division]   \n",
       "7                                     [Civil Division]   \n",
       "19   [Environment and Natural Resources Division, U...   \n",
       "22                                    [Civil Division]   \n",
       "23            [Civil Division, USAO - Florida, Middle]   \n",
       "..                                                 ...   \n",
       "316                                     [Tax Division]   \n",
       "318                                     [Tax Division]   \n",
       "321                      [Tax Division, USAO - Alaska]   \n",
       "322                                     [Tax Division]   \n",
       "324                                     [Tax Division]   \n",
       "\n",
       "                                          contents_new  \n",
       "4    U.S. Department Justice , U.S. Environmental P...  \n",
       "7    131 - count criminal indictment unsealed today...  \n",
       "19   United States Attorney Office Middle District ...  \n",
       "22   21st Century Oncology LLC , agreed pay $ 19.75...  \n",
       "23   21st Century Oncology Inc. certain subsidiarie...  \n",
       "..                                                 ...  \n",
       "316  Doctor Hid Millions Secret Accounts Panama Ala...  \n",
       "318  Defendant Concealed Bank Accounts Panama Costa...  \n",
       "321  Alaskan couple charged federal court Juneau , ...  \n",
       "322  husband wife pleaded guilty yesterday counts w...  \n",
       "324  resident Big Lake , Alaska indicted counts tax...  \n",
       "\n",
       "[100 rows x 7 columns]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6286"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.contents[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4810"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.contents_new[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The U.S. Department of Justice, the U.S. Environmental Protection Agency (EPA), and the Rhode Island Department of Environmental Management (RIDEM) announced today that two subsidiaries of Stanley Black & Decker Inc.—Emhart Industries Inc. and Black & Decker Inc.—have agreed to clean up dioxin conta'"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.contents[4][:300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'U.S. Department Justice , U.S. Environmental Protection Agency ( EPA ) , Rhode Island Department Environmental Management ( RIDEM ) announced today subsidiaries Stanley Black & Decker Inc.—Emhart Industries Inc. Black & Decker Inc.—have agreed clean dioxin contaminated sediment soil Centredale Manor'"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.contents_new[4][:300]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Examples where removing stop words can create a problem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**(1) Sentiment detection: Not always but in some cases, based on your dataset it can change the sentiment of a sentence if you remove stop words**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'good movie'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(\"this is a good movie\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'good movie'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(\"this is not a good movie\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**(2) Language translation: Say you want to translate following sentence from english to telugu. Before actual translation if you remove stop words and then translate, it will produce horrible result**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dhaval ?'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(\"how are you doing dhaval?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**(3) Chat bot or any Q&A system**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'find yoga mat website . help ?'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(\"I don't find yoga mat on your website. Can you help?\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "BOW_exercise.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}