3.4 Practical.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.4 Practical task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import spacy\n",
    "import re\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "bbc_data = pd.read_csv(\"bbc_news.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>index</th>\n",
       "      <th>title</th>\n",
       "      <th>pubDate</th>\n",
       "      <th>guid</th>\n",
       "      <th>link</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6684</td>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>Wed, 10 Aug 2022 15:46:18 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/business-62147992</td>\n",
       "      <td>https://www.bbc.co.uk/news/business-62147992?a...</td>\n",
       "      <td>With much of the UK enduring another period of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>9267</td>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>Mon, 17 Oct 2022 11:35:12 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/world-63285480</td>\n",
       "      <td>https://www.bbc.co.uk/news/world-63285480?at_m...</td>\n",
       "      <td>The UK's political chaos has been watched arou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>7387</td>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>Wed, 31 Aug 2022 05:20:18 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/uk-scotland-highlan...</td>\n",
       "      <td>https://www.bbc.co.uk/news/uk-scotland-highlan...</td>\n",
       "      <td>Scoraig in the north west Highlands has long h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>767</td>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>Tue, 22 Mar 2022 14:37:01 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/60739336</td>\n",
       "      <td>https://www.bbc.co.uk/news/60739336?at_medium=...</td>\n",
       "      <td>Wealthy Russians sanctioned by the US, EU and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3712</td>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>Wed, 01 Jun 2022 23:17:33 GMT</td>\n",
       "      <td>https://www.bbc.co.uk/news/uk-61660128</td>\n",
       "      <td>https://www.bbc.co.uk/news/uk-61660128?at_medi...</td>\n",
       "      <td>A quick look back at the Queen's 70 years on t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  index                                              title  \\\n",
       "0           0   6684                              Can I refuse to work?   \n",
       "1           1   9267  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2           2   7387  Rationing energy is nothing new for off-grid c...   \n",
       "3           3    767  The hunt for superyachts of sanctioned Russian...   \n",
       "4           4   3712  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                         pubDate  \\\n",
       "0  Wed, 10 Aug 2022 15:46:18 GMT   \n",
       "1  Mon, 17 Oct 2022 11:35:12 GMT   \n",
       "2  Wed, 31 Aug 2022 05:20:18 GMT   \n",
       "3  Tue, 22 Mar 2022 14:37:01 GMT   \n",
       "4  Wed, 01 Jun 2022 23:17:33 GMT   \n",
       "\n",
       "                                                guid  \\\n",
       "0       https://www.bbc.co.uk/news/business-62147992   \n",
       "1          https://www.bbc.co.uk/news/world-63285480   \n",
       "2  https://www.bbc.co.uk/news/uk-scotland-highlan...   \n",
       "3                https://www.bbc.co.uk/news/60739336   \n",
       "4             https://www.bbc.co.uk/news/uk-61660128   \n",
       "\n",
       "                                                link  \\\n",
       "0  https://www.bbc.co.uk/news/business-62147992?a...   \n",
       "1  https://www.bbc.co.uk/news/world-63285480?at_m...   \n",
       "2  https://www.bbc.co.uk/news/uk-scotland-highlan...   \n",
       "3  https://www.bbc.co.uk/news/60739336?at_medium=...   \n",
       "4  https://www.bbc.co.uk/news/uk-61660128?at_medi...   \n",
       "\n",
       "                                         description  \n",
       "0  With much of the UK enduring another period of...  \n",
       "1  The UK's political chaos has been watched arou...  \n",
       "2  Scoraig in the north west Highlands has long h...  \n",
       "3  Wealthy Russians sanctioned by the US, EU and ...  \n",
       "4  A quick look back at the Queen's 70 years on t...  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bbc_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1000 entries, 0 to 999\n",
      "Data columns (total 7 columns):\n",
      " #   Column       Non-Null Count  Dtype \n",
      "---  ------       --------------  ----- \n",
      " 0   Unnamed: 0   1000 non-null   int64 \n",
      " 1   index        1000 non-null   int64 \n",
      " 2   title        1000 non-null   object\n",
      " 3   pubDate      1000 non-null   object\n",
      " 4   guid         1000 non-null   object\n",
      " 5   link         1000 non-null   object\n",
      " 6   description  1000 non-null   object\n",
      "dtypes: int64(2), object(5)\n",
      "memory usage: 54.8+ KB\n"
     ]
    }
   ],
   "source": [
    "bbc_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title\n",
       "0                              Can I refuse to work?\n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...\n",
       "2  Rationing energy is nothing new for off-grid c...\n",
       "3  The hunt for superyachts of sanctioned Russian...\n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ..."
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles = pd.DataFrame(bbc_data['title'])\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \n",
       "0                              can i refuse to work?  \n",
       "1  'liz truss the brief?' world reacts to uk poli...  \n",
       "2  rationing energy is nothing new for off-grid c...  \n",
       "3  the hunt for superyachts of sanctioned russian...  \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lowercase\n",
    "titles['lowercase'] = titles['title'].str.lower()\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "      <th>no_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "      <td>refuse work?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "      <td>'liz truss brief?' world reacts uk political t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy nothing new off-grid community</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years queen 70 seconds</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \\\n",
       "0                              can i refuse to work?   \n",
       "1  'liz truss the brief?' world reacts to uk poli...   \n",
       "2  rationing energy is nothing new for off-grid c...   \n",
       "3  the hunt for superyachts of sanctioned russian...   \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...   \n",
       "\n",
       "                                        no_stopwords  \n",
       "0                                       refuse work?  \n",
       "1  'liz truss brief?' world reacts uk political t...  \n",
       "2    rationing energy nothing new off-grid community  \n",
       "3      hunt superyachts sanctioned russian oligarchs  \n",
       "4        platinum jubilee: 70 years queen 70 seconds  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stop word removal\n",
    "en_stopwords = stopwords.words('english')\n",
    "titles['no_stopwords'] = titles['lowercase'].apply(lambda x: ' '.join([word for word in x.split() if word not in (en_stopwords)]))\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "      <th>no_stopwords</th>\n",
       "      <th>no_stopwords_no_punct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "      <td>refuse work?</td>\n",
       "      <td>refuse work</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "      <td>'liz truss brief?' world reacts uk political t...</td>\n",
       "      <td>liz truss brief world reacts uk political turmoil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy nothing new off-grid community</td>\n",
       "      <td>rationing energy nothing new offgrid community</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years queen 70 seconds</td>\n",
       "      <td>platinum jubilee 70 years queen 70 seconds</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \\\n",
       "0                              can i refuse to work?   \n",
       "1  'liz truss the brief?' world reacts to uk poli...   \n",
       "2  rationing energy is nothing new for off-grid c...   \n",
       "3  the hunt for superyachts of sanctioned russian...   \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...   \n",
       "\n",
       "                                        no_stopwords  \\\n",
       "0                                       refuse work?   \n",
       "1  'liz truss brief?' world reacts uk political t...   \n",
       "2    rationing energy nothing new off-grid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4        platinum jubilee: 70 years queen 70 seconds   \n",
       "\n",
       "                               no_stopwords_no_punct  \n",
       "0                                        refuse work  \n",
       "1  liz truss brief world reacts uk political turmoil  \n",
       "2     rationing energy nothing new offgrid community  \n",
       "3      hunt superyachts sanctioned russian oligarchs  \n",
       "4         platinum jubilee 70 years queen 70 seconds  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# punctation removal\n",
    "titles['no_stopwords_no_punct'] = titles.apply(lambda x: re.sub(r\"([^\\w\\s])\", \"\", x['no_stopwords']), axis=1)\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "      <th>no_stopwords</th>\n",
       "      <th>no_stopwords_no_punct</th>\n",
       "      <th>tokens_raw</th>\n",
       "      <th>tokens_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "      <td>refuse work?</td>\n",
       "      <td>refuse work</td>\n",
       "      <td>[Can, I, refuse, to, work, ?]</td>\n",
       "      <td>[refuse, work]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "      <td>'liz truss brief?' world reacts uk political t...</td>\n",
       "      <td>liz truss brief world reacts uk political turmoil</td>\n",
       "      <td>['Liz, Truss, the, Brief, ?, ', World, reacts,...</td>\n",
       "      <td>[liz, truss, brief, world, reacts, uk, politic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy nothing new off-grid community</td>\n",
       "      <td>rationing energy nothing new offgrid community</td>\n",
       "      <td>[Rationing, energy, is, nothing, new, for, off...</td>\n",
       "      <td>[rationing, energy, nothing, new, offgrid, com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>[The, hunt, for, superyachts, of, sanctioned, ...</td>\n",
       "      <td>[hunt, superyachts, sanctioned, russian, oliga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years queen 70 seconds</td>\n",
       "      <td>platinum jubilee 70 years queen 70 seconds</td>\n",
       "      <td>[Platinum, Jubilee, :, 70, years, of, the, Que...</td>\n",
       "      <td>[platinum, jubilee, 70, years, queen, 70, seco...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \\\n",
       "0                              can i refuse to work?   \n",
       "1  'liz truss the brief?' world reacts to uk poli...   \n",
       "2  rationing energy is nothing new for off-grid c...   \n",
       "3  the hunt for superyachts of sanctioned russian...   \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...   \n",
       "\n",
       "                                        no_stopwords  \\\n",
       "0                                       refuse work?   \n",
       "1  'liz truss brief?' world reacts uk political t...   \n",
       "2    rationing energy nothing new off-grid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4        platinum jubilee: 70 years queen 70 seconds   \n",
       "\n",
       "                               no_stopwords_no_punct  \\\n",
       "0                                        refuse work   \n",
       "1  liz truss brief world reacts uk political turmoil   \n",
       "2     rationing energy nothing new offgrid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4         platinum jubilee 70 years queen 70 seconds   \n",
       "\n",
       "                                          tokens_raw  \\\n",
       "0                      [Can, I, refuse, to, work, ?]   \n",
       "1  ['Liz, Truss, the, Brief, ?, ', World, reacts,...   \n",
       "2  [Rationing, energy, is, nothing, new, for, off...   \n",
       "3  [The, hunt, for, superyachts, of, sanctioned, ...   \n",
       "4  [Platinum, Jubilee, :, 70, years, of, the, Que...   \n",
       "\n",
       "                                        tokens_clean  \n",
       "0                                     [refuse, work]  \n",
       "1  [liz, truss, brief, world, reacts, uk, politic...  \n",
       "2  [rationing, energy, nothing, new, offgrid, com...  \n",
       "3  [hunt, superyachts, sanctioned, russian, oliga...  \n",
       "4  [platinum, jubilee, 70, years, queen, 70, seco...  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tokenize\n",
    "titles['tokens_raw'] = titles.apply(lambda x: word_tokenize(x['title']), axis=1)\n",
    "titles['tokens_clean'] = titles.apply(lambda x: word_tokenize(x['no_stopwords_no_punct']), axis=1)\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>lowercase</th>\n",
       "      <th>no_stopwords</th>\n",
       "      <th>no_stopwords_no_punct</th>\n",
       "      <th>tokens_raw</th>\n",
       "      <th>tokens_clean</th>\n",
       "      <th>tokens_clean_lemmatized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Can I refuse to work?</td>\n",
       "      <td>can i refuse to work?</td>\n",
       "      <td>refuse work?</td>\n",
       "      <td>refuse work</td>\n",
       "      <td>[Can, I, refuse, to, work, ?]</td>\n",
       "      <td>[refuse, work]</td>\n",
       "      <td>[refuse, work]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'Liz Truss the Brief?' World reacts to UK poli...</td>\n",
       "      <td>'liz truss the brief?' world reacts to uk poli...</td>\n",
       "      <td>'liz truss brief?' world reacts uk political t...</td>\n",
       "      <td>liz truss brief world reacts uk political turmoil</td>\n",
       "      <td>['Liz, Truss, the, Brief, ?, ', World, reacts,...</td>\n",
       "      <td>[liz, truss, brief, world, reacts, uk, politic...</td>\n",
       "      <td>[liz, truss, brief, world, reacts, uk, politic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy is nothing new for off-grid c...</td>\n",
       "      <td>rationing energy nothing new off-grid community</td>\n",
       "      <td>rationing energy nothing new offgrid community</td>\n",
       "      <td>[Rationing, energy, is, nothing, new, for, off...</td>\n",
       "      <td>[rationing, energy, nothing, new, offgrid, com...</td>\n",
       "      <td>[rationing, energy, nothing, new, offgrid, com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The hunt for superyachts of sanctioned Russian...</td>\n",
       "      <td>the hunt for superyachts of sanctioned russian...</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>hunt superyachts sanctioned russian oligarchs</td>\n",
       "      <td>[The, hunt, for, superyachts, of, sanctioned, ...</td>\n",
       "      <td>[hunt, superyachts, sanctioned, russian, oliga...</td>\n",
       "      <td>[hunt, superyachts, sanctioned, russian, oliga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Platinum Jubilee: 70 years of the Queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years of the queen in 70 ...</td>\n",
       "      <td>platinum jubilee: 70 years queen 70 seconds</td>\n",
       "      <td>platinum jubilee 70 years queen 70 seconds</td>\n",
       "      <td>[Platinum, Jubilee, :, 70, years, of, the, Que...</td>\n",
       "      <td>[platinum, jubilee, 70, years, queen, 70, seco...</td>\n",
       "      <td>[platinum, jubilee, 70, year, queen, 70, second]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                              Can I refuse to work?   \n",
       "1  'Liz Truss the Brief?' World reacts to UK poli...   \n",
       "2  Rationing energy is nothing new for off-grid c...   \n",
       "3  The hunt for superyachts of sanctioned Russian...   \n",
       "4  Platinum Jubilee: 70 years of the Queen in 70 ...   \n",
       "\n",
       "                                           lowercase  \\\n",
       "0                              can i refuse to work?   \n",
       "1  'liz truss the brief?' world reacts to uk poli...   \n",
       "2  rationing energy is nothing new for off-grid c...   \n",
       "3  the hunt for superyachts of sanctioned russian...   \n",
       "4  platinum jubilee: 70 years of the queen in 70 ...   \n",
       "\n",
       "                                        no_stopwords  \\\n",
       "0                                       refuse work?   \n",
       "1  'liz truss brief?' world reacts uk political t...   \n",
       "2    rationing energy nothing new off-grid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4        platinum jubilee: 70 years queen 70 seconds   \n",
       "\n",
       "                               no_stopwords_no_punct  \\\n",
       "0                                        refuse work   \n",
       "1  liz truss brief world reacts uk political turmoil   \n",
       "2     rationing energy nothing new offgrid community   \n",
       "3      hunt superyachts sanctioned russian oligarchs   \n",
       "4         platinum jubilee 70 years queen 70 seconds   \n",
       "\n",
       "                                          tokens_raw  \\\n",
       "0                      [Can, I, refuse, to, work, ?]   \n",
       "1  ['Liz, Truss, the, Brief, ?, ', World, reacts,...   \n",
       "2  [Rationing, energy, is, nothing, new, for, off...   \n",
       "3  [The, hunt, for, superyachts, of, sanctioned, ...   \n",
       "4  [Platinum, Jubilee, :, 70, years, of, the, Que...   \n",
       "\n",
       "                                        tokens_clean  \\\n",
       "0                                     [refuse, work]   \n",
       "1  [liz, truss, brief, world, reacts, uk, politic...   \n",
       "2  [rationing, energy, nothing, new, offgrid, com...   \n",
       "3  [hunt, superyachts, sanctioned, russian, oliga...   \n",
       "4  [platinum, jubilee, 70, years, queen, 70, seco...   \n",
       "\n",
       "                             tokens_clean_lemmatized  \n",
       "0                                     [refuse, work]  \n",
       "1  [liz, truss, brief, world, reacts, uk, politic...  \n",
       "2  [rationing, energy, nothing, new, offgrid, com...  \n",
       "3  [hunt, superyachts, sanctioned, russian, oliga...  \n",
       "4   [platinum, jubilee, 70, year, queen, 70, second]  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lemmatizing \n",
    "lemmatizer = WordNetLemmatizer()\n",
    "titles[\"tokens_clean_lemmatized\"] = titles[\"tokens_clean\"].apply(lambda tokens: [lemmatizer.lemmatize(token) for token in tokens])\n",
    "titles.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create lists for just our tokens\n",
    "tokens_raw_list = sum(titles['tokens_raw'], []) #unpack our lists into a single list\n",
    "tokens_clean_list = sum(titles['tokens_clean_lemmatized'], [])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### POS Tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a spacy doc from our raw text - better for pos tagging\n",
    "spacy_doc = nlp(' '.join(tokens_raw_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract the tokens and pos tags into a dataframe\n",
    "pos_df = pd.DataFrame(columns=['token', 'pos_tag'])\n",
    "\n",
    "for token in spacy_doc:\n",
    "    pos_df = pos_df.append({'token': token.text,\n",
    "                            'pos_tag': token.pos_}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>:</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>'</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2897</th>\n",
       "      <td>in</td>\n",
       "      <td>ADP</td>\n",
       "      <td>188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3262</th>\n",
       "      <td>of</td>\n",
       "      <td>ADP</td>\n",
       "      <td>172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4075</th>\n",
       "      <td>to</td>\n",
       "      <td>PART</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>-</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4037</th>\n",
       "      <td>the</td>\n",
       "      <td>DET</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1861</th>\n",
       "      <td>and</td>\n",
       "      <td>CCONJ</td>\n",
       "      <td>147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>'s</td>\n",
       "      <td>PART</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>?</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     token pos_tag  counts\n",
       "94       :   PUNCT     543\n",
       "7        '   PUNCT     303\n",
       "2897    in     ADP     188\n",
       "3262    of     ADP     172\n",
       "4075    to    PART     171\n",
       "20       -   PUNCT     168\n",
       "4037   the     DET     163\n",
       "1861   and   CCONJ     147\n",
       "14      's    PART     142\n",
       "96       ?   PUNCT     130"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# token frequency count\n",
    "pos_df_counts = pos_df.groupby(['token','pos_tag']).size().reset_index(name='counts').sort_values(by='counts', ascending=False)\n",
    "pos_df_counts.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4259</th>\n",
       "      <td>war</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4347</th>\n",
       "      <td>year</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4307</th>\n",
       "      <td>win</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3410</th>\n",
       "      <td>police</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4003</th>\n",
       "      <td>tax</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3546</th>\n",
       "      <td>record</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3361</th>\n",
       "      <td>people</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2326</th>\n",
       "      <td>day</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3058</th>\n",
       "      <td>living</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2033</th>\n",
       "      <td>boss</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       token pos_tag  counts\n",
       "4259     war    NOUN      35\n",
       "4347    year    NOUN      14\n",
       "4307     win    NOUN      14\n",
       "3410  police    NOUN      14\n",
       "4003     tax    NOUN      13\n",
       "3546  record    NOUN      13\n",
       "3361  people    NOUN      12\n",
       "2326     day    NOUN      12\n",
       "3058  living    NOUN      12\n",
       "2033    boss    NOUN      11"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common nouns\n",
    "nouns = pos_df_counts[pos_df_counts.pos_tag == \"NOUN\"][0:10]\n",
    "nouns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3682</th>\n",
       "      <td>says</td>\n",
       "      <td>VERB</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2671</th>\n",
       "      <td>found</td>\n",
       "      <td>VERB</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4308</th>\n",
       "      <td>win</td>\n",
       "      <td>VERB</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4315</th>\n",
       "      <td>wins</td>\n",
       "      <td>VERB</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2713</th>\n",
       "      <td>get</td>\n",
       "      <td>VERB</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2979</th>\n",
       "      <td>killed</td>\n",
       "      <td>VERB</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3681</th>\n",
       "      <td>say</td>\n",
       "      <td>VERB</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1964</th>\n",
       "      <td>beat</td>\n",
       "      <td>VERB</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3740</th>\n",
       "      <td>set</td>\n",
       "      <td>VERB</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2386</th>\n",
       "      <td>dies</td>\n",
       "      <td>VERB</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       token pos_tag  counts\n",
       "3682    says    VERB      30\n",
       "2671   found    VERB      13\n",
       "4308     win    VERB      12\n",
       "4315    wins    VERB      10\n",
       "2713     get    VERB       9\n",
       "2979  killed    VERB       8\n",
       "3681     say    VERB       8\n",
       "1964    beat    VERB       8\n",
       "3740     set    VERB       8\n",
       "2386    dies    VERB       8"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common verbs\n",
    "verbs = pos_df_counts[pos_df_counts.pos_tag == \"VERB\"][0:10]\n",
    "verbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3239</th>\n",
       "      <td>new</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1404</th>\n",
       "      <td>Russian</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2607</th>\n",
       "      <td>final</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>-</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2626</th>\n",
       "      <td>first</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2997</th>\n",
       "      <td>last</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2834</th>\n",
       "      <td>high</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3194</th>\n",
       "      <td>more</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1995</th>\n",
       "      <td>big</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3297</th>\n",
       "      <td>other</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        token pos_tag  counts\n",
       "3239      new     ADJ      28\n",
       "1404  Russian     ADJ      22\n",
       "2607    final     ADJ      16\n",
       "18          -     ADJ      14\n",
       "2626    first     ADJ      12\n",
       "2997     last     ADJ      11\n",
       "2834     high     ADJ      10\n",
       "3194     more     ADJ      10\n",
       "1995      big     ADJ       9\n",
       "3297    other     ADJ       8"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common adjectives\n",
    "adj = pos_df_counts[pos_df_counts.pos_tag == \"ADJ\"][0:10]\n",
    "adj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract the tokens and entity tags into a dataframe\n",
    "ner_df = pd.DataFrame(columns=['token', 'ner_tag'])\n",
    "\n",
    "for token in spacy_doc.ents:\n",
    "    if pd.isna(token.label_) is False:\n",
    "        ner_df = ner_df.append({'token': token.text,\n",
    "                                'ner_tag': token.label_}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>ner_tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Liz Truss</td>\n",
       "      <td>PERSON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UK</td>\n",
       "      <td>GPE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Russian</td>\n",
       "      <td>NORP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70 years</td>\n",
       "      <td>DATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>70 seconds</td>\n",
       "      <td>TIME</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        token ner_tag\n",
       "0   Liz Truss  PERSON\n",
       "1          UK     GPE\n",
       "2     Russian    NORP\n",
       "3    70 years    DATE\n",
       "4  70 seconds    TIME"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>ner_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>904</th>\n",
       "      <td>Ukraine</td>\n",
       "      <td>GPE</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>310</th>\n",
       "      <td>England</td>\n",
       "      <td>GPE</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>UK</td>\n",
       "      <td>GPE</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>899</th>\n",
       "      <td>US</td>\n",
       "      <td>GPE</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>771</th>\n",
       "      <td>Russian</td>\n",
       "      <td>NORP</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>971</th>\n",
       "      <td>World Cup 2022</td>\n",
       "      <td>EVENT</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351</th>\n",
       "      <td>France</td>\n",
       "      <td>GPE</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>864</th>\n",
       "      <td>The Papers</td>\n",
       "      <td>WORK_OF_ART</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>China</td>\n",
       "      <td>GPE</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>first</td>\n",
       "      <td>ORDINAL</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              token      ner_tag  counts\n",
       "904         Ukraine          GPE      41\n",
       "310         England          GPE      36\n",
       "898              UK          GPE      32\n",
       "899              US          GPE      22\n",
       "771         Russian         NORP      20\n",
       "971  World Cup 2022        EVENT      18\n",
       "351          France          GPE      12\n",
       "864      The Papers  WORK_OF_ART      12\n",
       "205           China          GPE      10\n",
       "997           first      ORDINAL      10"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# token frequency count\n",
    "ner_df_counts = ner_df.groupby(['token','ner_tag']).size().reset_index(name='counts').sort_values(by='counts', ascending=False)\n",
    "ner_df_counts.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>ner_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>714</th>\n",
       "      <td>Putin</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531</th>\n",
       "      <td>Liz Truss</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>Covid</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>Boris Johnson</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>760</th>\n",
       "      <td>Rory McIlroy</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651</th>\n",
       "      <td>Nicola Bulley</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306</th>\n",
       "      <td>Emma Raducanu</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482</th>\n",
       "      <td>Jurgen Klopp</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>744</th>\n",
       "      <td>Rishi Sunak</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>888</th>\n",
       "      <td>Truss</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             token ner_tag  counts\n",
       "714          Putin  PERSON       8\n",
       "531      Liz Truss  PERSON       6\n",
       "235          Covid  PERSON       6\n",
       "150  Boris Johnson  PERSON       6\n",
       "760   Rory McIlroy  PERSON       4\n",
       "651  Nicola Bulley  PERSON       4\n",
       "306  Emma Raducanu  PERSON       4\n",
       "482   Jurgen Klopp  PERSON       4\n",
       "744    Rishi Sunak  PERSON       4\n",
       "888          Truss  PERSON       3"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common people\n",
    "people = ner_df_counts[ner_df_counts.ner_tag == \"PERSON\"][0:10]\n",
    "people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>token</th>\n",
       "      <th>ner_tag</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>904</th>\n",
       "      <td>Ukraine</td>\n",
       "      <td>GPE</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>310</th>\n",
       "      <td>England</td>\n",
       "      <td>GPE</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>UK</td>\n",
       "      <td>GPE</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>899</th>\n",
       "      <td>US</td>\n",
       "      <td>GPE</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351</th>\n",
       "      <td>France</td>\n",
       "      <td>GPE</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>China</td>\n",
       "      <td>GPE</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>769</th>\n",
       "      <td>Russia</td>\n",
       "      <td>GPE</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>India</td>\n",
       "      <td>GPE</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>535</th>\n",
       "      <td>London</td>\n",
       "      <td>GPE</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>Australia</td>\n",
       "      <td>GPE</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         token ner_tag  counts\n",
       "904    Ukraine     GPE      41\n",
       "310    England     GPE      36\n",
       "898         UK     GPE      32\n",
       "899         US     GPE      22\n",
       "351     France     GPE      12\n",
       "205      China     GPE      10\n",
       "769     Russia     GPE       9\n",
       "424      India     GPE       9\n",
       "535     London     GPE       7\n",
       "114  Australia     GPE       7"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common places\n",
    "places = ner_df_counts[ner_df_counts.ner_tag == \"GPE\"][0:10]\n",
    "places"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}