preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import ast\n",
    "import html\n",
    "import string\n",
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import wordnet\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "scr_1 = pd.read_csv('./datasets/raw/20210405_2330_vaccine.csv')\n",
    "scr_2 = pd.read_csv('./datasets/raw/20210406_2048_vaccine.csv')\n",
    "scr_3 = pd.read_csv('./datasets/raw/20210505_2259_vaccine.csv')\n",
    "scr_4 = pd.read_csv('./datasets/raw/20210507_0228_sinovac.csv')\n",
    "scr_5 = pd.read_csv('./datasets/raw/20210507_0618_sinovac.csv')\n",
    "scr_6 = pd.read_csv('./datasets/raw/20210507_1805_sinovac.csv')\n",
    "scr_7 = pd.read_csv('./datasets/raw/20210507_2140_biontech.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([scr_1, scr_2, scr_3, scr_4, scr_5, scr_6, scr_7], axis = 0, ignore_index=True)\n",
    "data.reset_index(inplace = True, drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "              username       location  following  followers  totaltweets  \\\n",
       "99995  JennellTebbett1            NaN        180         73        13959   \n",
       "99996    orig80saddict  United States       4963       2247        31045   \n",
       "99997       aiyanna_19            NaN        167        145          476   \n",
       "99998         DianeSzl            NaN         25          0          724   \n",
       "99999        LibAthe80       Room 237       2556       2415       189951   \n",
       "\n",
       "          whentweetcreated      whenacctcreated  retweets  \\\n",
       "99995  2021-05-07 15:08:47  2015-10-18 22:56:29      1073   \n",
       "99996  2021-05-07 15:08:44  2012-03-27 12:17:05         1   \n",
       "99997  2021-05-07 15:08:36  2019-11-26 21:52:45      1073   \n",
       "99998  2021-05-07 15:08:33  2020-05-12 15:40:10      1073   \n",
       "99999  2021-05-07 15:08:30  2011-02-07 20:57:57        10   \n",
       "\n",
       "                                                    text  \\\n",
       "99995  BREAKING: Today, with @BioNTech_Group, we init...   \n",
       "99996  You've been #vaccinated, and I've been #vaccin...   \n",
       "99997  BREAKING: Today, with @BioNTech_Group, we init...   \n",
       "99998  BREAKING: Today, with @BioNTech_Group, we init...   \n",
       "99999  A new RNA vaccine is in the works, and it may ...   \n",
       "\n",
       "                                                hashtags  \n",
       "99995       [{'text': 'COVID19', 'indices': [129, 137]}]  \n",
       "99996  [{'text': 'vaccinated', 'indices': [12, 23]}, ...  \n",
       "99997       [{'text': 'COVID19', 'indices': [129, 137]}]  \n",
       "99998       [{'text': 'COVID19', 'indices': [129, 137]}]  \n",
       "99999                                                 []  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>totaltweets</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>99995</th>\n      <td>JennellTebbett1</td>\n      <td>NaN</td>\n      <td>180</td>\n      <td>73</td>\n      <td>13959</td>\n      <td>2021-05-07 15:08:47</td>\n      <td>2015-10-18 22:56:29</td>\n      <td>1073</td>\n      <td>BREAKING: Today, with @BioNTech_Group, we init...</td>\n      <td>[{'text': 'COVID19', 'indices': [129, 137]}]</td>\n    </tr>\n    <tr>\n      <th>99996</th>\n      <td>orig80saddict</td>\n      <td>United States</td>\n      <td>4963</td>\n      <td>2247</td>\n      <td>31045</td>\n      <td>2021-05-07 15:08:44</td>\n      <td>2012-03-27 12:17:05</td>\n      <td>1</td>\n      <td>You've been #vaccinated, and I've been #vaccin...</td>\n      <td>[{'text': 'vaccinated', 'indices': [12, 23]}, ...</td>\n    </tr>\n    <tr>\n      <th>99997</th>\n      <td>aiyanna_19</td>\n      <td>NaN</td>\n      <td>167</td>\n      <td>145</td>\n      <td>476</td>\n      <td>2021-05-07 15:08:36</td>\n      <td>2019-11-26 21:52:45</td>\n      <td>1073</td>\n      <td>BREAKING: Today, with @BioNTech_Group, we init...</td>\n      <td>[{'text': 'COVID19', 'indices': [129, 137]}]</td>\n    </tr>\n    <tr>\n      <th>99998</th>\n      <td>DianeSzl</td>\n      <td>NaN</td>\n      <td>25</td>\n      <td>0</td>\n      <td>724</td>\n      <td>2021-05-07 15:08:33</td>\n      <td>2020-05-12 15:40:10</td>\n      <td>1073</td>\n      <td>BREAKING: Today, with @BioNTech_Group, we init...</td>\n      <td>[{'text': 'COVID19', 'indices': [129, 137]}]</td>\n    </tr>\n    <tr>\n      <th>99999</th>\n      <td>LibAthe80</td>\n      <td>Room 237</td>\n      <td>2556</td>\n      <td>2415</td>\n      <td>189951</td>\n      <td>2021-05-07 15:08:30</td>\n      <td>2011-02-07 20:57:57</td>\n      <td>10</td>\n      <td>A new RNA vaccine is in the works, and it may ...</td>\n      <td>[]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "username            100000\n",
       "location             68576\n",
       "following           100000\n",
       "followers           100000\n",
       "totaltweets         100000\n",
       "whentweetcreated    100000\n",
       "whenacctcreated     100000\n",
       "retweets            100000\n",
       "text                100000\n",
       "hashtags            100000\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop duplicated rows:\n",
    "data.drop_duplicates(keep=False, inplace=True)\n",
    "# Drop empty tweets and missing values\n",
    "data.dropna(subset=['text'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "username            66253\n",
       "location            44766\n",
       "following           66253\n",
       "followers           66253\n",
       "totaltweets         66253\n",
       "whentweetcreated    66253\n",
       "whenacctcreated     66253\n",
       "retweets            66253\n",
       "text                66253\n",
       "hashtags            66253\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.reset_index(inplace = True, drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "              username  location  following  followers  totaltweets  \\\n",
       "66248         bitficus  Flatland       1526        438         3857   \n",
       "66249  JennellTebbett1       NaN        180         73        13959   \n",
       "66250       aiyanna_19       NaN        167        145          476   \n",
       "66251         DianeSzl       NaN         25          0          724   \n",
       "66252        LibAthe80  Room 237       2556       2415       189951   \n",
       "\n",
       "          whentweetcreated      whenacctcreated  retweets  \\\n",
       "66248  2021-05-07 15:08:50  2018-02-06 06:29:49       107   \n",
       "66249  2021-05-07 15:08:47  2015-10-18 22:56:29      1073   \n",
       "66250  2021-05-07 15:08:36  2019-11-26 21:52:45      1073   \n",
       "66251  2021-05-07 15:08:33  2020-05-12 15:40:10      1073   \n",
       "66252  2021-05-07 15:08:30  2011-02-07 20:57:57        10   \n",
       "\n",
       "                                                    text  \\\n",
       "66248  The Pfizer-BioNTech COVID-19 Vaccine has not b...   \n",
       "66249  BREAKING: Today, with @BioNTech_Group, we init...   \n",
       "66250  BREAKING: Today, with @BioNTech_Group, we init...   \n",
       "66251  BREAKING: Today, with @BioNTech_Group, we init...   \n",
       "66252  A new RNA vaccine is in the works, and it may ...   \n",
       "\n",
       "                                           hashtags  \n",
       "66248                                            []  \n",
       "66249  [{'text': 'COVID19', 'indices': [129, 137]}]  \n",
       "66250  [{'text': 'COVID19', 'indices': [129, 137]}]  \n",
       "66251  [{'text': 'COVID19', 'indices': [129, 137]}]  \n",
       "66252                                            []  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>totaltweets</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>66248</th>\n      <td>bitficus</td>\n      <td>Flatland</td>\n      <td>1526</td>\n      <td>438</td>\n      <td>3857</td>\n      <td>2021-05-07 15:08:50</td>\n      <td>2018-02-06 06:29:49</td>\n      <td>107</td>\n      <td>The Pfizer-BioNTech COVID-19 Vaccine has not b...</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>66249</th>\n      <td>JennellTebbett1</td>\n      <td>NaN</td>\n      <td>180</td>\n      <td>73</td>\n      <td>13959</td>\n      <td>2021-05-07 15:08:47</td>\n      <td>2015-10-18 22:56:29</td>\n      <td>1073</td>\n      <td>BREAKING: Today, with @BioNTech_Group, we init...</td>\n      <td>[{'text': 'COVID19', 'indices': [129, 137]}]</td>\n    </tr>\n    <tr>\n      <th>66250</th>\n      <td>aiyanna_19</td>\n      <td>NaN</td>\n      <td>167</td>\n      <td>145</td>\n      <td>476</td>\n      <td>2021-05-07 15:08:36</td>\n      <td>2019-11-26 21:52:45</td>\n      <td>1073</td>\n      <td>BREAKING: Today, with @BioNTech_Group, we init...</td>\n      <td>[{'text': 'COVID19', 'indices': [129, 137]}]</td>\n    </tr>\n    <tr>\n      <th>66251</th>\n      <td>DianeSzl</td>\n      <td>NaN</td>\n      <td>25</td>\n      <td>0</td>\n      <td>724</td>\n      <td>2021-05-07 15:08:33</td>\n      <td>2020-05-12 15:40:10</td>\n      <td>1073</td>\n      <td>BREAKING: Today, with @BioNTech_Group, we init...</td>\n      <td>[{'text': 'COVID19', 'indices': [129, 137]}]</td>\n    </tr>\n    <tr>\n      <th>66252</th>\n      <td>LibAthe80</td>\n      <td>Room 237</td>\n      <td>2556</td>\n      <td>2415</td>\n      <td>189951</td>\n      <td>2021-05-07 15:08:30</td>\n      <td>2011-02-07 20:57:57</td>\n      <td>10</td>\n      <td>A new RNA vaccine is in the works, and it may ...</td>\n      <td>[]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.drop('totaltweets', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_list(text):\n",
    "    if pd.isnull(text):\n",
    "        text = ''\n",
    "    else:\n",
    "        text = ast.literal_eval(text)\n",
    "        # function to convert a string representation of a list to python list\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['hashtags'] = data['hashtags'].apply(lambda x: to_list(x))\n",
    "# tek bir tweetteki her hashtagin dict olduğu bir liste döndürür"
   ]
  },
  {
   "source": [
    "def extract_hashtags(hst_list):\n",
    "    hashtags = []\n",
    "    for i in range(0, len(hst_list)):\n",
    "        hashtags.append(hst_list[i]['text'].lower())\n",
    "    return hashtags"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 13,
   "outputs": []
  },
  {
   "source": [
    "data['hashtags'] = data['hashtags'].apply(lambda x: extract_hashtags(x))"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 14,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.reset_index(inplace = True, drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "              username  location  following  followers     whentweetcreated  \\\n",
       "66248         bitficus  Flatland       1526        438  2021-05-07 15:08:50   \n",
       "66249  JennellTebbett1       NaN        180         73  2021-05-07 15:08:47   \n",
       "66250       aiyanna_19       NaN        167        145  2021-05-07 15:08:36   \n",
       "66251         DianeSzl       NaN         25          0  2021-05-07 15:08:33   \n",
       "66252        LibAthe80  Room 237       2556       2415  2021-05-07 15:08:30   \n",
       "\n",
       "           whenacctcreated  retweets  \\\n",
       "66248  2018-02-06 06:29:49       107   \n",
       "66249  2015-10-18 22:56:29      1073   \n",
       "66250  2019-11-26 21:52:45      1073   \n",
       "66251  2020-05-12 15:40:10      1073   \n",
       "66252  2011-02-07 20:57:57        10   \n",
       "\n",
       "                                                    text   hashtags  \n",
       "66248  The Pfizer-BioNTech COVID-19 Vaccine has not b...         []  \n",
       "66249  BREAKING: Today, with @BioNTech_Group, we init...  [covid19]  \n",
       "66250  BREAKING: Today, with @BioNTech_Group, we init...  [covid19]  \n",
       "66251  BREAKING: Today, with @BioNTech_Group, we init...  [covid19]  \n",
       "66252  A new RNA vaccine is in the works, and it may ...         []  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>66248</th>\n      <td>bitficus</td>\n      <td>Flatland</td>\n      <td>1526</td>\n      <td>438</td>\n      <td>2021-05-07 15:08:50</td>\n      <td>2018-02-06 06:29:49</td>\n      <td>107</td>\n      <td>The Pfizer-BioNTech COVID-19 Vaccine has not b...</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>66249</th>\n      <td>JennellTebbett1</td>\n      <td>NaN</td>\n      <td>180</td>\n      <td>73</td>\n      <td>2021-05-07 15:08:47</td>\n      <td>2015-10-18 22:56:29</td>\n      <td>1073</td>\n      <td>BREAKING: Today, with @BioNTech_Group, we init...</td>\n      <td>[covid19]</td>\n    </tr>\n    <tr>\n      <th>66250</th>\n      <td>aiyanna_19</td>\n      <td>NaN</td>\n      <td>167</td>\n      <td>145</td>\n      <td>2021-05-07 15:08:36</td>\n      <td>2019-11-26 21:52:45</td>\n      <td>1073</td>\n      <td>BREAKING: Today, with @BioNTech_Group, we init...</td>\n      <td>[covid19]</td>\n    </tr>\n    <tr>\n      <th>66251</th>\n      <td>DianeSzl</td>\n      <td>NaN</td>\n      <td>25</td>\n      <td>0</td>\n      <td>2021-05-07 15:08:33</td>\n      <td>2020-05-12 15:40:10</td>\n      <td>1073</td>\n      <td>BREAKING: Today, with @BioNTech_Group, we init...</td>\n      <td>[covid19]</td>\n    </tr>\n    <tr>\n      <th>66252</th>\n      <td>LibAthe80</td>\n      <td>Room 237</td>\n      <td>2556</td>\n      <td>2415</td>\n      <td>2021-05-07 15:08:30</td>\n      <td>2011-02-07 20:57:57</td>\n      <td>10</td>\n      <td>A new RNA vaccine is in the works, and it may ...</td>\n      <td>[]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "scr_8 = pd.read_csv('./datasets/raw/biontech.csv')\n",
    "scr_9 = pd.read_csv('./datasets/raw/sinovac.csv')\n",
    "scr_10 = pd.read_csv('./datasets/raw/astrazeneca.csv')\n",
    "scr_11 = pd.read_csv('./datasets/raw/moderna.csv')\n",
    "scr_12 = pd.read_csv('./datasets/raw/sputnik.csv')\n",
    "dummy = pd.concat([scr_8, scr_9, scr_10, scr_11, scr_12], axis=0, ignore_index=False)\n",
    "dummy.reset_index(inplace = True, drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "Unnamed: 0        4287\n",
       "id                3866\n",
       "user_name         4287\n",
       "user_location     4287\n",
       "user_created      4287\n",
       "user_followers    4287\n",
       "user_friends      4287\n",
       "date              4287\n",
       "text              4287\n",
       "hashtags          4287\n",
       "retweets          4287\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "dummy.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "      Unnamed: 0  id                                       user_name  \\\n",
       "4282       69578 NaN                              Carlos André Puras   \n",
       "4283       69616 NaN                                         Mustafa   \n",
       "4284       69619 NaN                                             Sue   \n",
       "4285       69668 NaN  UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS   \n",
       "4286       69705 NaN                                    Naga Sridhar   \n",
       "\n",
       "          user_location         user_created  user_followers  user_friends  \\\n",
       "4282  São Paulo, Brasil  2009-10-19 15:29:13             418          1051   \n",
       "4283              india  2014-08-13 17:53:08            1474          5014   \n",
       "4284            Lebanon  2011-09-17 19:22:58             392           614   \n",
       "4285   New Delhi, India  2017-04-09 09:31:10            1291             6   \n",
       "4286          Hyderabad  2011-09-29 07:54:56             668           675   \n",
       "\n",
       "                     date                                               text  \\\n",
       "4282  2021-04-14 16:36:20  @sputnikvaccine #SputnikV  #FechadoComBolsonar...   \n",
       "4283  2021-04-14 15:08:55  Putin’s Once-Scorned Vaccine Now Favorite in P...   \n",
       "4284  2021-04-14 14:54:04  Today I got my first dose of the #SputnikV cor...   \n",
       "4285  2021-04-14 12:01:03  Healing mankind! \\n#SputnikV\\n#Sputnikvaccine ...   \n",
       "4286  2021-04-14 08:47:39  #sputnikv #DRREDDYS #COVID # Sputinik V will b...   \n",
       "\n",
       "                                               hashtags  retweets  \n",
       "4282  ['SputnikV', 'FechadoComBolsonaro', 'STF', 'Do...         0  \n",
       "4283                                       ['SputnikV']         0  \n",
       "4284  ['SputnikV', 'vaccines', 'CoronaVaccine', 'vac...         0  \n",
       "4285  ['SputnikV', 'Sputnikvaccine', 'sputnikvaccine...        45  \n",
       "4286                  ['sputnikv', 'DRREDDYS', 'COVID']         0  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Unnamed: 0</th>\n      <th>id</th>\n      <th>user_name</th>\n      <th>user_location</th>\n      <th>user_created</th>\n      <th>user_followers</th>\n      <th>user_friends</th>\n      <th>date</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>retweets</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>4282</th>\n      <td>69578</td>\n      <td>NaN</td>\n      <td>Carlos André Puras</td>\n      <td>São Paulo, Brasil</td>\n      <td>2009-10-19 15:29:13</td>\n      <td>418</td>\n      <td>1051</td>\n      <td>2021-04-14 16:36:20</td>\n      <td>@sputnikvaccine #SputnikV  #FechadoComBolsonar...</td>\n      <td>['SputnikV', 'FechadoComBolsonaro', 'STF', 'Do...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4283</th>\n      <td>69616</td>\n      <td>NaN</td>\n      <td>Mustafa</td>\n      <td>india</td>\n      <td>2014-08-13 17:53:08</td>\n      <td>1474</td>\n      <td>5014</td>\n      <td>2021-04-14 15:08:55</td>\n      <td>Putin’s Once-Scorned Vaccine Now Favorite in P...</td>\n      <td>['SputnikV']</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4284</th>\n      <td>69619</td>\n      <td>NaN</td>\n      <td>Sue</td>\n      <td>Lebanon</td>\n      <td>2011-09-17 19:22:58</td>\n      <td>392</td>\n      <td>614</td>\n      <td>2021-04-14 14:54:04</td>\n      <td>Today I got my first dose of the #SputnikV cor...</td>\n      <td>['SputnikV', 'vaccines', 'CoronaVaccine', 'vac...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4285</th>\n      <td>69668</td>\n      <td>NaN</td>\n      <td>UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS</td>\n      <td>New Delhi, India</td>\n      <td>2017-04-09 09:31:10</td>\n      <td>1291</td>\n      <td>6</td>\n      <td>2021-04-14 12:01:03</td>\n      <td>Healing mankind! \\n#SputnikV\\n#Sputnikvaccine ...</td>\n      <td>['SputnikV', 'Sputnikvaccine', 'sputnikvaccine...</td>\n      <td>45</td>\n    </tr>\n    <tr>\n      <th>4286</th>\n      <td>69705</td>\n      <td>NaN</td>\n      <td>Naga Sridhar</td>\n      <td>Hyderabad</td>\n      <td>2011-09-29 07:54:56</td>\n      <td>668</td>\n      <td>675</td>\n      <td>2021-04-14 08:47:39</td>\n      <td>#sputnikv #DRREDDYS #COVID # Sputinik V will b...</td>\n      <td>['sputnikv', 'DRREDDYS', 'COVID']</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "dummy.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy = dummy.drop('Unnamed: 0', axis=1)\n",
    "dummy = dummy.drop('id', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "switch = {'user_name': 'username', 'user_location': 'location',\n",
    "            'user_created': 'whenacctcreated', 'user_followers': 'followers',\n",
    "            'user_friends': 'following', 'date': 'whentweetcreated',\n",
    "        }\n",
    "dummy = dummy.rename(columns=switch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names = ['username', 'location', 'following', 'followers', 'whentweetcreated',\n",
    "            'whenacctcreated', 'retweets', 'text', 'hashtags']\n",
    "dummy = dummy.reindex(columns=column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                                            username           location  \\\n",
       "4282                              Carlos André Puras  São Paulo, Brasil   \n",
       "4283                                         Mustafa              india   \n",
       "4284                                             Sue            Lebanon   \n",
       "4285  UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS   New Delhi, India   \n",
       "4286                                    Naga Sridhar          Hyderabad   \n",
       "\n",
       "      following  followers     whentweetcreated      whenacctcreated  \\\n",
       "4282       1051        418  2021-04-14 16:36:20  2009-10-19 15:29:13   \n",
       "4283       5014       1474  2021-04-14 15:08:55  2014-08-13 17:53:08   \n",
       "4284        614        392  2021-04-14 14:54:04  2011-09-17 19:22:58   \n",
       "4285          6       1291  2021-04-14 12:01:03  2017-04-09 09:31:10   \n",
       "4286        675        668  2021-04-14 08:47:39  2011-09-29 07:54:56   \n",
       "\n",
       "      retweets                                               text  \\\n",
       "4282         0  @sputnikvaccine #SputnikV  #FechadoComBolsonar...   \n",
       "4283         0  Putin’s Once-Scorned Vaccine Now Favorite in P...   \n",
       "4284         0  Today I got my first dose of the #SputnikV cor...   \n",
       "4285        45  Healing mankind! \\n#SputnikV\\n#Sputnikvaccine ...   \n",
       "4286         0  #sputnikv #DRREDDYS #COVID # Sputinik V will b...   \n",
       "\n",
       "                                               hashtags  \n",
       "4282  ['SputnikV', 'FechadoComBolsonaro', 'STF', 'Do...  \n",
       "4283                                       ['SputnikV']  \n",
       "4284  ['SputnikV', 'vaccines', 'CoronaVaccine', 'vac...  \n",
       "4285  ['SputnikV', 'Sputnikvaccine', 'sputnikvaccine...  \n",
       "4286                  ['sputnikv', 'DRREDDYS', 'COVID']  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>4282</th>\n      <td>Carlos André Puras</td>\n      <td>São Paulo, Brasil</td>\n      <td>1051</td>\n      <td>418</td>\n      <td>2021-04-14 16:36:20</td>\n      <td>2009-10-19 15:29:13</td>\n      <td>0</td>\n      <td>@sputnikvaccine #SputnikV  #FechadoComBolsonar...</td>\n      <td>['SputnikV', 'FechadoComBolsonaro', 'STF', 'Do...</td>\n    </tr>\n    <tr>\n      <th>4283</th>\n      <td>Mustafa</td>\n      <td>india</td>\n      <td>5014</td>\n      <td>1474</td>\n      <td>2021-04-14 15:08:55</td>\n      <td>2014-08-13 17:53:08</td>\n      <td>0</td>\n      <td>Putin’s Once-Scorned Vaccine Now Favorite in P...</td>\n      <td>['SputnikV']</td>\n    </tr>\n    <tr>\n      <th>4284</th>\n      <td>Sue</td>\n      <td>Lebanon</td>\n      <td>614</td>\n      <td>392</td>\n      <td>2021-04-14 14:54:04</td>\n      <td>2011-09-17 19:22:58</td>\n      <td>0</td>\n      <td>Today I got my first dose of the #SputnikV cor...</td>\n      <td>['SputnikV', 'vaccines', 'CoronaVaccine', 'vac...</td>\n    </tr>\n    <tr>\n      <th>4285</th>\n      <td>UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS</td>\n      <td>New Delhi, India</td>\n      <td>6</td>\n      <td>1291</td>\n      <td>2021-04-14 12:01:03</td>\n      <td>2017-04-09 09:31:10</td>\n      <td>45</td>\n      <td>Healing mankind! \\n#SputnikV\\n#Sputnikvaccine ...</td>\n      <td>['SputnikV', 'Sputnikvaccine', 'sputnikvaccine...</td>\n    </tr>\n    <tr>\n      <th>4286</th>\n      <td>Naga Sridhar</td>\n      <td>Hyderabad</td>\n      <td>675</td>\n      <td>668</td>\n      <td>2021-04-14 08:47:39</td>\n      <td>2011-09-29 07:54:56</td>\n      <td>0</td>\n      <td>#sputnikv #DRREDDYS #COVID # Sputinik V will b...</td>\n      <td>['sputnikv', 'DRREDDYS', 'COVID']</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "dummy.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([data, dummy], axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "username            70540\n",
       "location            49053\n",
       "following           70540\n",
       "followers           70540\n",
       "whentweetcreated    70540\n",
       "whenacctcreated     70540\n",
       "retweets            70540\n",
       "text                70540\n",
       "hashtags            70540\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 25
    }
   ],
   "source": [
    "data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                                             username           location  \\\n",
       "70535                              Carlos André Puras  São Paulo, Brasil   \n",
       "70536                                         Mustafa              india   \n",
       "70537                                             Sue            Lebanon   \n",
       "70538  UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS   New Delhi, India   \n",
       "70539                                    Naga Sridhar          Hyderabad   \n",
       "\n",
       "       following  followers     whentweetcreated      whenacctcreated  \\\n",
       "70535       1051        418  2021-04-14 16:36:20  2009-10-19 15:29:13   \n",
       "70536       5014       1474  2021-04-14 15:08:55  2014-08-13 17:53:08   \n",
       "70537        614        392  2021-04-14 14:54:04  2011-09-17 19:22:58   \n",
       "70538          6       1291  2021-04-14 12:01:03  2017-04-09 09:31:10   \n",
       "70539        675        668  2021-04-14 08:47:39  2011-09-29 07:54:56   \n",
       "\n",
       "       retweets                                               text  \\\n",
       "70535         0  @sputnikvaccine #SputnikV  #FechadoComBolsonar...   \n",
       "70536         0  Putin’s Once-Scorned Vaccine Now Favorite in P...   \n",
       "70537         0  Today I got my first dose of the #SputnikV cor...   \n",
       "70538        45  Healing mankind! \\n#SputnikV\\n#Sputnikvaccine ...   \n",
       "70539         0  #sputnikv #DRREDDYS #COVID # Sputinik V will b...   \n",
       "\n",
       "                                                hashtags  \n",
       "70535  ['SputnikV', 'FechadoComBolsonaro', 'STF', 'Do...  \n",
       "70536                                       ['SputnikV']  \n",
       "70537  ['SputnikV', 'vaccines', 'CoronaVaccine', 'vac...  \n",
       "70538  ['SputnikV', 'Sputnikvaccine', 'sputnikvaccine...  \n",
       "70539                  ['sputnikv', 'DRREDDYS', 'COVID']  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>70535</th>\n      <td>Carlos André Puras</td>\n      <td>São Paulo, Brasil</td>\n      <td>1051</td>\n      <td>418</td>\n      <td>2021-04-14 16:36:20</td>\n      <td>2009-10-19 15:29:13</td>\n      <td>0</td>\n      <td>@sputnikvaccine #SputnikV  #FechadoComBolsonar...</td>\n      <td>['SputnikV', 'FechadoComBolsonaro', 'STF', 'Do...</td>\n    </tr>\n    <tr>\n      <th>70536</th>\n      <td>Mustafa</td>\n      <td>india</td>\n      <td>5014</td>\n      <td>1474</td>\n      <td>2021-04-14 15:08:55</td>\n      <td>2014-08-13 17:53:08</td>\n      <td>0</td>\n      <td>Putin’s Once-Scorned Vaccine Now Favorite in P...</td>\n      <td>['SputnikV']</td>\n    </tr>\n    <tr>\n      <th>70537</th>\n      <td>Sue</td>\n      <td>Lebanon</td>\n      <td>614</td>\n      <td>392</td>\n      <td>2021-04-14 14:54:04</td>\n      <td>2011-09-17 19:22:58</td>\n      <td>0</td>\n      <td>Today I got my first dose of the #SputnikV cor...</td>\n      <td>['SputnikV', 'vaccines', 'CoronaVaccine', 'vac...</td>\n    </tr>\n    <tr>\n      <th>70538</th>\n      <td>UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS</td>\n      <td>New Delhi, India</td>\n      <td>6</td>\n      <td>1291</td>\n      <td>2021-04-14 12:01:03</td>\n      <td>2017-04-09 09:31:10</td>\n      <td>45</td>\n      <td>Healing mankind! \\n#SputnikV\\n#Sputnikvaccine ...</td>\n      <td>['SputnikV', 'Sputnikvaccine', 'sputnikvaccine...</td>\n    </tr>\n    <tr>\n      <th>70539</th>\n      <td>Naga Sridhar</td>\n      <td>Hyderabad</td>\n      <td>675</td>\n      <td>668</td>\n      <td>2021-04-14 08:47:39</td>\n      <td>2011-09-29 07:54:56</td>\n      <td>0</td>\n      <td>#sputnikv #DRREDDYS #COVID # Sputinik V will b...</td>\n      <td>['sputnikv', 'DRREDDYS', 'COVID']</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 26
    }
   ],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collects mentioned users (@username)\n",
    "def mentioned_users(string):\n",
    "    usernames = re.findall('@[^\\s]+', string)\n",
    "    return usernames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "         username               location  following  followers  \\\n",
       "0   LoloCraiger29                    NaN        274        188   \n",
       "1     Nexusswagon             Illinois         5002       1591   \n",
       "2  mariemueller41                    NaN        726        370   \n",
       "3    BettyBunny23  Clotted cream first.        2866       2674   \n",
       "4         dwm2112           Galt's Gulch        350        163   \n",
       "\n",
       "      whentweetcreated      whenacctcreated  retweets  \\\n",
       "0  2021-04-05 20:28:21  2012-09-26 20:45:27         1   \n",
       "1  2021-04-05 20:28:20  2018-12-13 13:46:54      3185   \n",
       "2  2021-04-05 20:28:20  2010-12-24 03:17:41         0   \n",
       "3  2021-04-05 20:28:20  2015-06-01 20:54:44         3   \n",
       "4  2021-04-05 20:28:20  2009-04-28 19:44:48      5902   \n",
       "\n",
       "                                                text hashtags  \\\n",
       "0  How many mom's out there have vaccination reco...       []   \n",
       "1  6+ months ago, I said that governments would p...       []   \n",
       "2  @jbirdbradyII My freshman year of college I wa...       []   \n",
       "3  @Nigel_Farage I am slightly confused we we wer...       []   \n",
       "4  Requiring Americans to show proof of vaccinati...       []   \n",
       "\n",
       "            mentioned_users  \n",
       "0                        []  \n",
       "1                        []  \n",
       "2  [@jbirdbradyII, @drdrew]  \n",
       "3           [@Nigel_Farage]  \n",
       "4                        []  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>mentioned_users</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>LoloCraiger29</td>\n      <td>NaN</td>\n      <td>274</td>\n      <td>188</td>\n      <td>2021-04-05 20:28:21</td>\n      <td>2012-09-26 20:45:27</td>\n      <td>1</td>\n      <td>How many mom's out there have vaccination reco...</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Nexusswagon</td>\n      <td>Illinois</td>\n      <td>5002</td>\n      <td>1591</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2018-12-13 13:46:54</td>\n      <td>3185</td>\n      <td>6+ months ago, I said that governments would p...</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>mariemueller41</td>\n      <td>NaN</td>\n      <td>726</td>\n      <td>370</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2010-12-24 03:17:41</td>\n      <td>0</td>\n      <td>@jbirdbradyII My freshman year of college I wa...</td>\n      <td>[]</td>\n      <td>[@jbirdbradyII, @drdrew]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>BettyBunny23</td>\n      <td>Clotted cream first.</td>\n      <td>2866</td>\n      <td>2674</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2015-06-01 20:54:44</td>\n      <td>3</td>\n      <td>@Nigel_Farage I am slightly confused we we wer...</td>\n      <td>[]</td>\n      <td>[@Nigel_Farage]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>dwm2112</td>\n      <td>Galt's Gulch</td>\n      <td>350</td>\n      <td>163</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2009-04-28 19:44:48</td>\n      <td>5902</td>\n      <td>Requiring Americans to show proof of vaccinati...</td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 28
    }
   ],
   "source": [
    "# Create a new column and apply the function on the column 'text'\n",
    "data['mentioned_users'] = data['text'].apply(lambda x: mentioned_users(x))\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the function to implement POS tagging:\n",
    "def get_wordnet_pos(pos_tag):\n",
    "    if pos_tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif pos_tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif pos_tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif pos_tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleaning(text):\n",
    "    # remove html chars\n",
    "    text = html.unescape(text)\n",
    "\n",
    "    # remove urls\n",
    "    text = re.sub('((www\\.[^\\s]+)|(https?://[^\\s]+))', '', text)\n",
    "\n",
    "    # remove usernames\n",
    "    text = re.sub('@[^\\s]+', '', text)\n",
    "\n",
    "    # remove hashtags\n",
    "    text = re.sub(r'#([^\\s]+)', '', text)\n",
    "\n",
    "    # remove &amp;\n",
    "    text = text.replace('&amp;', 'and')\n",
    "\n",
    "    # remove punctuations and lower text\n",
    "    text = text.lower()\n",
    "    text = [word.strip(string.punctuation) for word in text.split(\" \")]\n",
    "\n",
    "    # remove numbers\n",
    "    text = [word for word in text if not any(c.isdigit() for c in word)]\n",
    "\n",
    "    # remove stop words\n",
    "    stop = stopwords.words('english')\n",
    "    text = [x for x in text if x not in stop]\n",
    "\n",
    "    # remove empty tokens\n",
    "    text = [t for t in text if len(t) > 0]\n",
    "\n",
    "    # pos tag text and lemmatize text\n",
    "    pos_tags = pos_tag(text)\n",
    "    text = [WordNetLemmatizer().lemmatize(t[0], get_wordnet_pos(t[1])) for t in pos_tags]\n",
    "\n",
    "    # remove words with only one letter\n",
    "    text = [t for t in text if len(t) > 1]\n",
    "\n",
    "    # join all\n",
    "    text = \" \".join(text)\n",
    "        \n",
    "    return(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['cleaned_text'] = data['text'].apply(lambda x: cleaning(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                                             username           location  \\\n",
       "70535                              Carlos André Puras  São Paulo, Brasil   \n",
       "70536                                         Mustafa              india   \n",
       "70537                                             Sue            Lebanon   \n",
       "70538  UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS   New Delhi, India   \n",
       "70539                                    Naga Sridhar          Hyderabad   \n",
       "\n",
       "       following  followers     whentweetcreated      whenacctcreated  \\\n",
       "70535       1051        418  2021-04-14 16:36:20  2009-10-19 15:29:13   \n",
       "70536       5014       1474  2021-04-14 15:08:55  2014-08-13 17:53:08   \n",
       "70537        614        392  2021-04-14 14:54:04  2011-09-17 19:22:58   \n",
       "70538          6       1291  2021-04-14 12:01:03  2017-04-09 09:31:10   \n",
       "70539        675        668  2021-04-14 08:47:39  2011-09-29 07:54:56   \n",
       "\n",
       "       retweets                                               text  \\\n",
       "70535         0  @sputnikvaccine #SputnikV  #FechadoComBolsonar...   \n",
       "70536         0  Putin’s Once-Scorned Vaccine Now Favorite in P...   \n",
       "70537         0  Today I got my first dose of the #SputnikV cor...   \n",
       "70538        45  Healing mankind! \\n#SputnikV\\n#Sputnikvaccine ...   \n",
       "70539         0  #sputnikv #DRREDDYS #COVID # Sputinik V will b...   \n",
       "\n",
       "                                                hashtags  \\\n",
       "70535  ['SputnikV', 'FechadoComBolsonaro', 'STF', 'Do...   \n",
       "70536                                       ['SputnikV']   \n",
       "70537  ['SputnikV', 'vaccines', 'CoronaVaccine', 'vac...   \n",
       "70538  ['SputnikV', 'Sputnikvaccine', 'sputnikvaccine...   \n",
       "70539                  ['sputnikv', 'DRREDDYS', 'COVID']   \n",
       "\n",
       "                    mentioned_users  \\\n",
       "70535             [@sputnikvaccine]   \n",
       "70536  [@drreddys, @sputnikvaccine]   \n",
       "70537             [@sputnikvaccine]   \n",
       "70538                            []   \n",
       "70539                            []   \n",
       "\n",
       "                                            cleaned_text  \n",
       "70535            brazilian supreme court communists try…  \n",
       "70536  putin’s once-scorned vaccine favorite pandemic...  \n",
       "70537        today get first dose coronavirus vaccine.\\n  \n",
       "70538                                  heal mankind \\n\\n  \n",
       "70539  sputinik available india current quarter dr re...  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>mentioned_users</th>\n      <th>cleaned_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>70535</th>\n      <td>Carlos André Puras</td>\n      <td>São Paulo, Brasil</td>\n      <td>1051</td>\n      <td>418</td>\n      <td>2021-04-14 16:36:20</td>\n      <td>2009-10-19 15:29:13</td>\n      <td>0</td>\n      <td>@sputnikvaccine #SputnikV  #FechadoComBolsonar...</td>\n      <td>['SputnikV', 'FechadoComBolsonaro', 'STF', 'Do...</td>\n      <td>[@sputnikvaccine]</td>\n      <td>brazilian supreme court communists try…</td>\n    </tr>\n    <tr>\n      <th>70536</th>\n      <td>Mustafa</td>\n      <td>india</td>\n      <td>5014</td>\n      <td>1474</td>\n      <td>2021-04-14 15:08:55</td>\n      <td>2014-08-13 17:53:08</td>\n      <td>0</td>\n      <td>Putin’s Once-Scorned Vaccine Now Favorite in P...</td>\n      <td>['SputnikV']</td>\n      <td>[@drreddys, @sputnikvaccine]</td>\n      <td>putin’s once-scorned vaccine favorite pandemic...</td>\n    </tr>\n    <tr>\n      <th>70537</th>\n      <td>Sue</td>\n      <td>Lebanon</td>\n      <td>614</td>\n      <td>392</td>\n      <td>2021-04-14 14:54:04</td>\n      <td>2011-09-17 19:22:58</td>\n      <td>0</td>\n      <td>Today I got my first dose of the #SputnikV cor...</td>\n      <td>['SputnikV', 'vaccines', 'CoronaVaccine', 'vac...</td>\n      <td>[@sputnikvaccine]</td>\n      <td>today get first dose coronavirus vaccine.\\n</td>\n    </tr>\n    <tr>\n      <th>70538</th>\n      <td>UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS</td>\n      <td>New Delhi, India</td>\n      <td>6</td>\n      <td>1291</td>\n      <td>2021-04-14 12:01:03</td>\n      <td>2017-04-09 09:31:10</td>\n      <td>45</td>\n      <td>Healing mankind! \\n#SputnikV\\n#Sputnikvaccine ...</td>\n      <td>['SputnikV', 'Sputnikvaccine', 'sputnikvaccine...</td>\n      <td>[]</td>\n      <td>heal mankind \\n\\n</td>\n    </tr>\n    <tr>\n      <th>70539</th>\n      <td>Naga Sridhar</td>\n      <td>Hyderabad</td>\n      <td>675</td>\n      <td>668</td>\n      <td>2021-04-14 08:47:39</td>\n      <td>2011-09-29 07:54:56</td>\n      <td>0</td>\n      <td>#sputnikv #DRREDDYS #COVID # Sputinik V will b...</td>\n      <td>['sputnikv', 'DRREDDYS', 'COVID']</td>\n      <td>[]</td>\n      <td>sputinik available india current quarter dr re...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 32
    }
   ],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import preprocessor as p\n",
    "\n",
    "def preprocess_tweet(row):\n",
    "    p.set_options(p.OPT.EMOJI, p.OPT.SMILEY, p.OPT.RESERVED)\n",
    "    text = row['cleaned_text']\n",
    "    text = p.clean(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#apply the function to text column, axis = 1 for all rows\n",
    "data['cleaned_text'] = data.apply(preprocess_tweet, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0    many mom's vaccination record child big deal v...\n",
       "1    month ago say government would push vaccine pa...\n",
       "2    freshman year college wasnt allow enroll class...\n",
       "3    slightly confuse tell vaccine end life back no...\n",
       "4    require american show proof vaccination vaccin...\n",
       "5    tomorrow morning a.m state health official hol...\n",
       "6    maryland expand vaccine eligibility adult next...\n",
       "7    fed lead poll majority convince anyone robyn f...\n",
       "8    public health england advise pregnant woman pl...\n",
       "9                   sign help find vaccine appointment\n",
       "Name: cleaned_text, dtype: object"
      ]
     },
     "metadata": {},
     "execution_count": 35
    }
   ],
   "source": [
    "data.head(10)['cleaned_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "70530               brazilian supreme court communists try\n",
       "70531               brazilian supreme court communists try\n",
       "70532               brazilian supreme court communists try\n",
       "70533               brazilian supreme court communists try\n",
       "70534               brazilian supreme court communists try\n",
       "70535               brazilian supreme court communists try\n",
       "70536    putins once-scorned vaccine favorite pandemic ...\n",
       "70537            today get first dose coronavirus vaccine.\n",
       "70538                                         heal mankind\n",
       "70539    sputinik available india current quarter dr re...\n",
       "Name: cleaned_text, dtype: object"
      ]
     },
     "metadata": {},
     "execution_count": 36
    }
   ],
   "source": [
    "data.tail(10)['cleaned_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop duplicated rows:\n",
    "data.drop_duplicates(subset='cleaned_text', keep=\"first\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "70508                    need questionable foremost political tool div\n",
       "70509                           sec finally approve sputnik vaccine!!!\n",
       "70510                                          please help brazil need\n",
       "70511                      brazilian supreme court communists try dest\n",
       "70520                         brazilian supreme court communists tryin\n",
       "70529                           brazilian supreme court communists try\n",
       "70536    putins once-scorned vaccine favorite pandemic fight bloomberg\n",
       "70537                        today get first dose coronavirus vaccine.\n",
       "70538                                                     heal mankind\n",
       "70539              sputinik available india current quarter dr reddy's\n",
       "Name: cleaned_text, dtype: object"
      ]
     },
     "metadata": {},
     "execution_count": 39
    }
   ],
   "source": [
    "data.tail(10)['cleaned_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "username            24083\n",
       "location            18074\n",
       "following           24083\n",
       "followers           24083\n",
       "whentweetcreated    24083\n",
       "whenacctcreated     24083\n",
       "retweets            24083\n",
       "text                24083\n",
       "hashtags            24083\n",
       "mentioned_users     24083\n",
       "cleaned_text        24083\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 40
    }
   ],
   "source": [
    "data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(24083, 11)\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                                             username           location  \\\n",
       "24078                              Carlos André Puras  São Paulo, Brasil   \n",
       "24079                                         Mustafa              india   \n",
       "24080                                             Sue            Lebanon   \n",
       "24081  UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS   New Delhi, India   \n",
       "24082                                    Naga Sridhar          Hyderabad   \n",
       "\n",
       "       following  followers     whentweetcreated      whenacctcreated  \\\n",
       "24078       1051        418  2021-04-14 16:37:16  2009-10-19 15:29:13   \n",
       "24079       5014       1474  2021-04-14 15:08:55  2014-08-13 17:53:08   \n",
       "24080        614        392  2021-04-14 14:54:04  2011-09-17 19:22:58   \n",
       "24081          6       1291  2021-04-14 12:01:03  2017-04-09 09:31:10   \n",
       "24082        675        668  2021-04-14 08:47:39  2011-09-29 07:54:56   \n",
       "\n",
       "       retweets  \\\n",
       "24078         1   \n",
       "24079         0   \n",
       "24080         0   \n",
       "24081        45   \n",
       "24082         0   \n",
       "\n",
       "                                                                                                                                                  text  \\\n",
       "24078     @sputnikvaccine #SputnikV  #FechadoComBolsonaro  #STF   #DoriaPatife Brazilian Supreme Court and Communists are try… https://t.co/SmmjcHAXBh   \n",
       "24079             Putin’s Once-Scorned Vaccine Now Favorite in Pandemic Fight - Bloomberg @drreddys @sputnikvaccine #SputnikV… https://t.co/EMyVMjJ9Jj   \n",
       "24080      Today I got my first dose of the #SputnikV coronavirus vaccine.\\n@sputnikvaccine #vaccines #CoronaVaccine #vaccine… https://t.co/LHtjGBGpyL   \n",
       "24081  Healing mankind! \\n#SputnikV\\n#Sputnikvaccine \\n#sputnikvaccinelifesaver \\n#COVID19 \\n#CoronavirusIndia \\n#COVIDUpdates https://t.co/DXnwCls8G4   \n",
       "24082                                            #sputnikv #DRREDDYS #COVID # Sputinik V will be available in India in the current quarter: Dr Reddy's   \n",
       "\n",
       "                                                                                                       hashtags  \\\n",
       "24078                                                 ['SputnikV', 'FechadoComBolsonaro', 'STF', 'DoriaPatife']   \n",
       "24079                                                                                              ['SputnikV']   \n",
       "24080                                                      ['SputnikV', 'vaccines', 'CoronaVaccine', 'vaccine']   \n",
       "24081  ['SputnikV', 'Sputnikvaccine', 'sputnikvaccinelifesaver', 'COVID19', 'CoronavirusIndia', 'COVIDUpdates']   \n",
       "24082                                                                         ['sputnikv', 'DRREDDYS', 'COVID']   \n",
       "\n",
       "                    mentioned_users  \\\n",
       "24078             [@sputnikvaccine]   \n",
       "24079  [@drreddys, @sputnikvaccine]   \n",
       "24080             [@sputnikvaccine]   \n",
       "24081                            []   \n",
       "24082                            []   \n",
       "\n",
       "                                                        cleaned_text  \n",
       "24078                         brazilian supreme court communists try  \n",
       "24079  putins once-scorned vaccine favorite pandemic fight bloomberg  \n",
       "24080                      today get first dose coronavirus vaccine.  \n",
       "24081                                                   heal mankind  \n",
       "24082            sputinik available india current quarter dr reddy's  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>mentioned_users</th>\n      <th>cleaned_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>24078</th>\n      <td>Carlos André Puras</td>\n      <td>São Paulo, Brasil</td>\n      <td>1051</td>\n      <td>418</td>\n      <td>2021-04-14 16:37:16</td>\n      <td>2009-10-19 15:29:13</td>\n      <td>1</td>\n      <td>@sputnikvaccine #SputnikV  #FechadoComBolsonaro  #STF   #DoriaPatife Brazilian Supreme Court and Communists are try… https://t.co/SmmjcHAXBh</td>\n      <td>['SputnikV', 'FechadoComBolsonaro', 'STF', 'DoriaPatife']</td>\n      <td>[@sputnikvaccine]</td>\n      <td>brazilian supreme court communists try</td>\n    </tr>\n    <tr>\n      <th>24079</th>\n      <td>Mustafa</td>\n      <td>india</td>\n      <td>5014</td>\n      <td>1474</td>\n      <td>2021-04-14 15:08:55</td>\n      <td>2014-08-13 17:53:08</td>\n      <td>0</td>\n      <td>Putin’s Once-Scorned Vaccine Now Favorite in Pandemic Fight - Bloomberg @drreddys @sputnikvaccine #SputnikV… https://t.co/EMyVMjJ9Jj</td>\n      <td>['SputnikV']</td>\n      <td>[@drreddys, @sputnikvaccine]</td>\n      <td>putins once-scorned vaccine favorite pandemic fight bloomberg</td>\n    </tr>\n    <tr>\n      <th>24080</th>\n      <td>Sue</td>\n      <td>Lebanon</td>\n      <td>614</td>\n      <td>392</td>\n      <td>2021-04-14 14:54:04</td>\n      <td>2011-09-17 19:22:58</td>\n      <td>0</td>\n      <td>Today I got my first dose of the #SputnikV coronavirus vaccine.\\n@sputnikvaccine #vaccines #CoronaVaccine #vaccine… https://t.co/LHtjGBGpyL</td>\n      <td>['SputnikV', 'vaccines', 'CoronaVaccine', 'vaccine']</td>\n      <td>[@sputnikvaccine]</td>\n      <td>today get first dose coronavirus vaccine.</td>\n    </tr>\n    <tr>\n      <th>24081</th>\n      <td>UNNATI - THE KNOWLEDGE HUB FOR CIVIL ASPIRANTS</td>\n      <td>New Delhi, India</td>\n      <td>6</td>\n      <td>1291</td>\n      <td>2021-04-14 12:01:03</td>\n      <td>2017-04-09 09:31:10</td>\n      <td>45</td>\n      <td>Healing mankind! \\n#SputnikV\\n#Sputnikvaccine \\n#sputnikvaccinelifesaver \\n#COVID19 \\n#CoronavirusIndia \\n#COVIDUpdates https://t.co/DXnwCls8G4</td>\n      <td>['SputnikV', 'Sputnikvaccine', 'sputnikvaccinelifesaver', 'COVID19', 'CoronavirusIndia', 'COVIDUpdates']</td>\n      <td>[]</td>\n      <td>heal mankind</td>\n    </tr>\n    <tr>\n      <th>24082</th>\n      <td>Naga Sridhar</td>\n      <td>Hyderabad</td>\n      <td>675</td>\n      <td>668</td>\n      <td>2021-04-14 08:47:39</td>\n      <td>2011-09-29 07:54:56</td>\n      <td>0</td>\n      <td>#sputnikv #DRREDDYS #COVID # Sputinik V will be available in India in the current quarter: Dr Reddy's</td>\n      <td>['sputnikv', 'DRREDDYS', 'COVID']</td>\n      <td>[]</td>\n      <td>sputinik available india current quarter dr reddy's</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 41
    }
   ],
   "source": [
    "# Check out the shape again and reset_index\n",
    "print(data.shape)\n",
    "data.reset_index(inplace = True, drop = True)\n",
    "\n",
    "# Check out data.tail() to validate index has been reset\n",
    "data.tail()"
   ]
  },
  {
   "source": [
    "# LOCATIONS"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_nan(text):\n",
    "    if pd.isnull(text):\n",
    "        text = 'n.a.'\n",
    "    else:\n",
    "        text = text.lower()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['location'] = data['location'].apply(lambda x:remove_nan(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "n.a.                  6009\n",
       "united states          324\n",
       "los angeles, ca        204\n",
       "new york, ny           203\n",
       "canada                 174\n",
       "                      ... \n",
       "nairobi.kenya            1\n",
       "atx/nyc                  1\n",
       "essex girl in kent       1\n",
       "south east asia          1\n",
       "in transit               1\n",
       "Name: location, Length: 7976, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 44
    }
   ],
   "source": [
    "data['location'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "sid = SentimentIntensityAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['sent_by_lexicon'] = data['cleaned_text'].apply(lambda x: sid.polarity_scores(x))\n",
    "\n",
    "# Keep only the compound scores under the column 'Sentiment'\n",
    "data = pd.concat([data.drop(['sent_by_lexicon'], axis = 1), data['sent_by_lexicon'].apply(pd.Series)], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "         username               location  following  followers  \\\n",
       "0   LoloCraiger29                   n.a.        274        188   \n",
       "1     Nexusswagon             illinois         5002       1591   \n",
       "2  mariemueller41                   n.a.        726        370   \n",
       "3    BettyBunny23  clotted cream first.        2866       2674   \n",
       "4         dwm2112           galt's gulch        350        163   \n",
       "\n",
       "      whentweetcreated      whenacctcreated  retweets  \\\n",
       "0  2021-04-05 20:28:21  2012-09-26 20:45:27         1   \n",
       "1  2021-04-05 20:28:20  2018-12-13 13:46:54      3185   \n",
       "2  2021-04-05 20:28:20  2010-12-24 03:17:41         0   \n",
       "3  2021-04-05 20:28:20  2015-06-01 20:54:44         3   \n",
       "4  2021-04-05 20:28:20  2009-04-28 19:44:48      5902   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                         text  \\\n",
       "0                 How many mom's out there have vaccination records for your children? What is the big deal with vaccination passports? They show you have had the vaccine and also you have resect for people with underlining conditions. I guess that's why Republicans are carrying on!!!   \n",
       "1      6+ months ago, I said that governments would push for 'vaccine passports' or 'freedom passes' as part of 'The New Normal/Great Reset'.\\n\\nI was told that I was a 'crazy conspiracy theorist'.\\n\\nNow, those same people are trying to convince me that it's a good idea. \\n\\nMuppets.   \n",
       "2  @jbirdbradyII My freshman year of college I wasn’t allowed to enroll in classes because my 2nd MMR vaccine was given when I was 360 days old and not 1 year. So at 18 I got a baby vaccine. Also @drdrew is an intentional idiot. He just needed some attention today. Here it is sweetie.   \n",
       "3       @Nigel_Farage I am slightly confused we we were told the vaccine was the be all and end all !! Life back to normal so why are they bringing in the vaccine passport when 90% of the country would’ve been vaccinated ? It sounds like they were not quite as confident as they were .   \n",
       "4                    Requiring Americans to show proof of vaccination, or a ‘vaccine passport,’ in order to engage in society is outrageous and un-American. This isn’t about containing a virus, but is just another dangerous idea from the Left in their quest for more power and control.   \n",
       "\n",
       "  hashtags           mentioned_users  \\\n",
       "0       []                        []   \n",
       "1       []                        []   \n",
       "2       []  [@jbirdbradyII, @drdrew]   \n",
       "3       []           [@Nigel_Farage]   \n",
       "4       []                        []   \n",
       "\n",
       "                                                                                                                                                                   cleaned_text  \\\n",
       "0                        many mom's vaccination record child big deal vaccination passports show vaccine also resect people underlining condition guess that's republican carry   \n",
       "1  month ago say government would push vaccine passport freedom pass part new normal/great reset'. i tell crazy conspiracy theorist'. now people try convince good idea muppets   \n",
       "2                             freshman year college wasnt allow enroll class mmr vaccine give day old year get baby vaccine also intentional idiot need attention today sweetie   \n",
       "3                                            slightly confuse tell vaccine end life back normal bringing vaccine passport country wouldve vaccinated sound like quite confident   \n",
       "4     require american show proof vaccination vaccine passport, order engage society outrageous un-american isnt contain virus another dangerous idea leave quest power control   \n",
       "\n",
       "     neg    neu    pos  compound  \n",
       "0  0.000  1.000  0.000    0.0000  \n",
       "1  0.166  0.573  0.261    0.5106  \n",
       "2  0.176  0.710  0.114   -0.1940  \n",
       "3  0.071  0.664  0.265    0.6582  \n",
       "4  0.264  0.650  0.087   -0.5994  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>mentioned_users</th>\n      <th>cleaned_text</th>\n      <th>neg</th>\n      <th>neu</th>\n      <th>pos</th>\n      <th>compound</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>LoloCraiger29</td>\n      <td>n.a.</td>\n      <td>274</td>\n      <td>188</td>\n      <td>2021-04-05 20:28:21</td>\n      <td>2012-09-26 20:45:27</td>\n      <td>1</td>\n      <td>How many mom's out there have vaccination records for your children? What is the big deal with vaccination passports? They show you have had the vaccine and also you have resect for people with underlining conditions. I guess that's why Republicans are carrying on!!!</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>many mom's vaccination record child big deal vaccination passports show vaccine also resect people underlining condition guess that's republican carry</td>\n      <td>0.000</td>\n      <td>1.000</td>\n      <td>0.000</td>\n      <td>0.0000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Nexusswagon</td>\n      <td>illinois</td>\n      <td>5002</td>\n      <td>1591</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2018-12-13 13:46:54</td>\n      <td>3185</td>\n      <td>6+ months ago, I said that governments would push for 'vaccine passports' or 'freedom passes' as part of 'The New Normal/Great Reset'.\\n\\nI was told that I was a 'crazy conspiracy theorist'.\\n\\nNow, those same people are trying to convince me that it's a good idea. \\n\\nMuppets.</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>month ago say government would push vaccine passport freedom pass part new normal/great reset'. i tell crazy conspiracy theorist'. now people try convince good idea muppets</td>\n      <td>0.166</td>\n      <td>0.573</td>\n      <td>0.261</td>\n      <td>0.5106</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>mariemueller41</td>\n      <td>n.a.</td>\n      <td>726</td>\n      <td>370</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2010-12-24 03:17:41</td>\n      <td>0</td>\n      <td>@jbirdbradyII My freshman year of college I wasn’t allowed to enroll in classes because my 2nd MMR vaccine was given when I was 360 days old and not 1 year. So at 18 I got a baby vaccine. Also @drdrew is an intentional idiot. He just needed some attention today. Here it is sweetie.</td>\n      <td>[]</td>\n      <td>[@jbirdbradyII, @drdrew]</td>\n      <td>freshman year college wasnt allow enroll class mmr vaccine give day old year get baby vaccine also intentional idiot need attention today sweetie</td>\n      <td>0.176</td>\n      <td>0.710</td>\n      <td>0.114</td>\n      <td>-0.1940</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>BettyBunny23</td>\n      <td>clotted cream first.</td>\n      <td>2866</td>\n      <td>2674</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2015-06-01 20:54:44</td>\n      <td>3</td>\n      <td>@Nigel_Farage I am slightly confused we we were told the vaccine was the be all and end all !! Life back to normal so why are they bringing in the vaccine passport when 90% of the country would’ve been vaccinated ? It sounds like they were not quite as confident as they were .</td>\n      <td>[]</td>\n      <td>[@Nigel_Farage]</td>\n      <td>slightly confuse tell vaccine end life back normal bringing vaccine passport country wouldve vaccinated sound like quite confident</td>\n      <td>0.071</td>\n      <td>0.664</td>\n      <td>0.265</td>\n      <td>0.6582</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>dwm2112</td>\n      <td>galt's gulch</td>\n      <td>350</td>\n      <td>163</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2009-04-28 19:44:48</td>\n      <td>5902</td>\n      <td>Requiring Americans to show proof of vaccination, or a ‘vaccine passport,’ in order to engage in society is outrageous and un-American. This isn’t about containing a virus, but is just another dangerous idea from the Left in their quest for more power and control.</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>require american show proof vaccination vaccine passport, order engage society outrageous un-american isnt contain virus another dangerous idea leave quest power control</td>\n      <td>0.264</td>\n      <td>0.650</td>\n      <td>0.087</td>\n      <td>-0.5994</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 47
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new column called 'sentiment_class'\n",
    "sentimentclass_list = []\n",
    "\n",
    "for i in range(0, len(data)):\n",
    "    \n",
    "    # current 'compound' score:\n",
    "    curr_compound = data.iloc[i,:]['compound']\n",
    "    \n",
    "    if (curr_compound <= 1.0 and curr_compound >= 0.10):\n",
    "        sentimentclass_list.append(1)\n",
    "    elif (curr_compound < 0.10 and curr_compound > -0.10):\n",
    "        sentimentclass_list.append(0)\n",
    "    elif (curr_compound <= -0.10 and curr_compound >= -1.00):\n",
    "        sentimentclass_list.append(-1)\n",
    "\n",
    "# Add the new column 'sentiment_class' to the dataframe\n",
    "data['sentiment'] = sentimentclass_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "         username               location  following  followers  \\\n",
       "0   LoloCraiger29                   n.a.        274        188   \n",
       "1     Nexusswagon             illinois         5002       1591   \n",
       "2  mariemueller41                   n.a.        726        370   \n",
       "3    BettyBunny23  clotted cream first.        2866       2674   \n",
       "4         dwm2112           galt's gulch        350        163   \n",
       "\n",
       "      whentweetcreated      whenacctcreated  retweets  \\\n",
       "0  2021-04-05 20:28:21  2012-09-26 20:45:27         1   \n",
       "1  2021-04-05 20:28:20  2018-12-13 13:46:54      3185   \n",
       "2  2021-04-05 20:28:20  2010-12-24 03:17:41         0   \n",
       "3  2021-04-05 20:28:20  2015-06-01 20:54:44         3   \n",
       "4  2021-04-05 20:28:20  2009-04-28 19:44:48      5902   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                         text  \\\n",
       "0                 How many mom's out there have vaccination records for your children? What is the big deal with vaccination passports? They show you have had the vaccine and also you have resect for people with underlining conditions. I guess that's why Republicans are carrying on!!!   \n",
       "1      6+ months ago, I said that governments would push for 'vaccine passports' or 'freedom passes' as part of 'The New Normal/Great Reset'.\\n\\nI was told that I was a 'crazy conspiracy theorist'.\\n\\nNow, those same people are trying to convince me that it's a good idea. \\n\\nMuppets.   \n",
       "2  @jbirdbradyII My freshman year of college I wasn’t allowed to enroll in classes because my 2nd MMR vaccine was given when I was 360 days old and not 1 year. So at 18 I got a baby vaccine. Also @drdrew is an intentional idiot. He just needed some attention today. Here it is sweetie.   \n",
       "3       @Nigel_Farage I am slightly confused we we were told the vaccine was the be all and end all !! Life back to normal so why are they bringing in the vaccine passport when 90% of the country would’ve been vaccinated ? It sounds like they were not quite as confident as they were .   \n",
       "4                    Requiring Americans to show proof of vaccination, or a ‘vaccine passport,’ in order to engage in society is outrageous and un-American. This isn’t about containing a virus, but is just another dangerous idea from the Left in their quest for more power and control.   \n",
       "\n",
       "  hashtags           mentioned_users  \\\n",
       "0       []                        []   \n",
       "1       []                        []   \n",
       "2       []  [@jbirdbradyII, @drdrew]   \n",
       "3       []           [@Nigel_Farage]   \n",
       "4       []                        []   \n",
       "\n",
       "                                                                                                                                                                   cleaned_text  \\\n",
       "0                        many mom's vaccination record child big deal vaccination passports show vaccine also resect people underlining condition guess that's republican carry   \n",
       "1  month ago say government would push vaccine passport freedom pass part new normal/great reset'. i tell crazy conspiracy theorist'. now people try convince good idea muppets   \n",
       "2                             freshman year college wasnt allow enroll class mmr vaccine give day old year get baby vaccine also intentional idiot need attention today sweetie   \n",
       "3                                            slightly confuse tell vaccine end life back normal bringing vaccine passport country wouldve vaccinated sound like quite confident   \n",
       "4     require american show proof vaccination vaccine passport, order engage society outrageous un-american isnt contain virus another dangerous idea leave quest power control   \n",
       "\n",
       "     neg    neu    pos  compound  sentiment  \n",
       "0  0.000  1.000  0.000    0.0000          0  \n",
       "1  0.166  0.573  0.261    0.5106          1  \n",
       "2  0.176  0.710  0.114   -0.1940         -1  \n",
       "3  0.071  0.664  0.265    0.6582          1  \n",
       "4  0.264  0.650  0.087   -0.5994         -1  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>mentioned_users</th>\n      <th>cleaned_text</th>\n      <th>neg</th>\n      <th>neu</th>\n      <th>pos</th>\n      <th>compound</th>\n      <th>sentiment</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>LoloCraiger29</td>\n      <td>n.a.</td>\n      <td>274</td>\n      <td>188</td>\n      <td>2021-04-05 20:28:21</td>\n      <td>2012-09-26 20:45:27</td>\n      <td>1</td>\n      <td>How many mom's out there have vaccination records for your children? What is the big deal with vaccination passports? They show you have had the vaccine and also you have resect for people with underlining conditions. I guess that's why Republicans are carrying on!!!</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>many mom's vaccination record child big deal vaccination passports show vaccine also resect people underlining condition guess that's republican carry</td>\n      <td>0.000</td>\n      <td>1.000</td>\n      <td>0.000</td>\n      <td>0.0000</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Nexusswagon</td>\n      <td>illinois</td>\n      <td>5002</td>\n      <td>1591</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2018-12-13 13:46:54</td>\n      <td>3185</td>\n      <td>6+ months ago, I said that governments would push for 'vaccine passports' or 'freedom passes' as part of 'The New Normal/Great Reset'.\\n\\nI was told that I was a 'crazy conspiracy theorist'.\\n\\nNow, those same people are trying to convince me that it's a good idea. \\n\\nMuppets.</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>month ago say government would push vaccine passport freedom pass part new normal/great reset'. i tell crazy conspiracy theorist'. now people try convince good idea muppets</td>\n      <td>0.166</td>\n      <td>0.573</td>\n      <td>0.261</td>\n      <td>0.5106</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>mariemueller41</td>\n      <td>n.a.</td>\n      <td>726</td>\n      <td>370</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2010-12-24 03:17:41</td>\n      <td>0</td>\n      <td>@jbirdbradyII My freshman year of college I wasn’t allowed to enroll in classes because my 2nd MMR vaccine was given when I was 360 days old and not 1 year. So at 18 I got a baby vaccine. Also @drdrew is an intentional idiot. He just needed some attention today. Here it is sweetie.</td>\n      <td>[]</td>\n      <td>[@jbirdbradyII, @drdrew]</td>\n      <td>freshman year college wasnt allow enroll class mmr vaccine give day old year get baby vaccine also intentional idiot need attention today sweetie</td>\n      <td>0.176</td>\n      <td>0.710</td>\n      <td>0.114</td>\n      <td>-0.1940</td>\n      <td>-1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>BettyBunny23</td>\n      <td>clotted cream first.</td>\n      <td>2866</td>\n      <td>2674</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2015-06-01 20:54:44</td>\n      <td>3</td>\n      <td>@Nigel_Farage I am slightly confused we we were told the vaccine was the be all and end all !! Life back to normal so why are they bringing in the vaccine passport when 90% of the country would’ve been vaccinated ? It sounds like they were not quite as confident as they were .</td>\n      <td>[]</td>\n      <td>[@Nigel_Farage]</td>\n      <td>slightly confuse tell vaccine end life back normal bringing vaccine passport country wouldve vaccinated sound like quite confident</td>\n      <td>0.071</td>\n      <td>0.664</td>\n      <td>0.265</td>\n      <td>0.6582</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>dwm2112</td>\n      <td>galt's gulch</td>\n      <td>350</td>\n      <td>163</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2009-04-28 19:44:48</td>\n      <td>5902</td>\n      <td>Requiring Americans to show proof of vaccination, or a ‘vaccine passport,’ in order to engage in society is outrageous and un-American. This isn’t about containing a virus, but is just another dangerous idea from the Left in their quest for more power and control.</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>require american show proof vaccination vaccine passport, order engage society outrageous un-american isnt contain virus another dangerous idea leave quest power control</td>\n      <td>0.264</td>\n      <td>0.650</td>\n      <td>0.087</td>\n      <td>-0.5994</td>\n      <td>-1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 49
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.drop('neg', axis=1)\n",
    "data = data.drop('neu', axis=1)\n",
    "data = data.drop('pos', axis=1)\n",
    "data = data.drop('compound', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "         username               location  following  followers  \\\n",
       "0   LoloCraiger29                   n.a.        274        188   \n",
       "1     Nexusswagon             illinois         5002       1591   \n",
       "2  mariemueller41                   n.a.        726        370   \n",
       "3    BettyBunny23  clotted cream first.        2866       2674   \n",
       "4         dwm2112           galt's gulch        350        163   \n",
       "\n",
       "      whentweetcreated      whenacctcreated  retweets  \\\n",
       "0  2021-04-05 20:28:21  2012-09-26 20:45:27         1   \n",
       "1  2021-04-05 20:28:20  2018-12-13 13:46:54      3185   \n",
       "2  2021-04-05 20:28:20  2010-12-24 03:17:41         0   \n",
       "3  2021-04-05 20:28:20  2015-06-01 20:54:44         3   \n",
       "4  2021-04-05 20:28:20  2009-04-28 19:44:48      5902   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                         text  \\\n",
       "0                 How many mom's out there have vaccination records for your children? What is the big deal with vaccination passports? They show you have had the vaccine and also you have resect for people with underlining conditions. I guess that's why Republicans are carrying on!!!   \n",
       "1      6+ months ago, I said that governments would push for 'vaccine passports' or 'freedom passes' as part of 'The New Normal/Great Reset'.\\n\\nI was told that I was a 'crazy conspiracy theorist'.\\n\\nNow, those same people are trying to convince me that it's a good idea. \\n\\nMuppets.   \n",
       "2  @jbirdbradyII My freshman year of college I wasn’t allowed to enroll in classes because my 2nd MMR vaccine was given when I was 360 days old and not 1 year. So at 18 I got a baby vaccine. Also @drdrew is an intentional idiot. He just needed some attention today. Here it is sweetie.   \n",
       "3       @Nigel_Farage I am slightly confused we we were told the vaccine was the be all and end all !! Life back to normal so why are they bringing in the vaccine passport when 90% of the country would’ve been vaccinated ? It sounds like they were not quite as confident as they were .   \n",
       "4                    Requiring Americans to show proof of vaccination, or a ‘vaccine passport,’ in order to engage in society is outrageous and un-American. This isn’t about containing a virus, but is just another dangerous idea from the Left in their quest for more power and control.   \n",
       "\n",
       "  hashtags           mentioned_users  \\\n",
       "0       []                        []   \n",
       "1       []                        []   \n",
       "2       []  [@jbirdbradyII, @drdrew]   \n",
       "3       []           [@Nigel_Farage]   \n",
       "4       []                        []   \n",
       "\n",
       "                                                                                                                                                                   cleaned_text  \\\n",
       "0                        many mom's vaccination record child big deal vaccination passports show vaccine also resect people underlining condition guess that's republican carry   \n",
       "1  month ago say government would push vaccine passport freedom pass part new normal/great reset'. i tell crazy conspiracy theorist'. now people try convince good idea muppets   \n",
       "2                             freshman year college wasnt allow enroll class mmr vaccine give day old year get baby vaccine also intentional idiot need attention today sweetie   \n",
       "3                                            slightly confuse tell vaccine end life back normal bringing vaccine passport country wouldve vaccinated sound like quite confident   \n",
       "4     require american show proof vaccination vaccine passport, order engage society outrageous un-american isnt contain virus another dangerous idea leave quest power control   \n",
       "\n",
       "  sentiment  \n",
       "0   Neutral  \n",
       "1  Positive  \n",
       "2  Negative  \n",
       "3  Positive  \n",
       "4  Negative  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>username</th>\n      <th>location</th>\n      <th>following</th>\n      <th>followers</th>\n      <th>whentweetcreated</th>\n      <th>whenacctcreated</th>\n      <th>retweets</th>\n      <th>text</th>\n      <th>hashtags</th>\n      <th>mentioned_users</th>\n      <th>cleaned_text</th>\n      <th>sentiment</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>LoloCraiger29</td>\n      <td>n.a.</td>\n      <td>274</td>\n      <td>188</td>\n      <td>2021-04-05 20:28:21</td>\n      <td>2012-09-26 20:45:27</td>\n      <td>1</td>\n      <td>How many mom's out there have vaccination records for your children? What is the big deal with vaccination passports? They show you have had the vaccine and also you have resect for people with underlining conditions. I guess that's why Republicans are carrying on!!!</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>many mom's vaccination record child big deal vaccination passports show vaccine also resect people underlining condition guess that's republican carry</td>\n      <td>Neutral</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Nexusswagon</td>\n      <td>illinois</td>\n      <td>5002</td>\n      <td>1591</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2018-12-13 13:46:54</td>\n      <td>3185</td>\n      <td>6+ months ago, I said that governments would push for 'vaccine passports' or 'freedom passes' as part of 'The New Normal/Great Reset'.\\n\\nI was told that I was a 'crazy conspiracy theorist'.\\n\\nNow, those same people are trying to convince me that it's a good idea. \\n\\nMuppets.</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>month ago say government would push vaccine passport freedom pass part new normal/great reset'. i tell crazy conspiracy theorist'. now people try convince good idea muppets</td>\n      <td>Positive</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>mariemueller41</td>\n      <td>n.a.</td>\n      <td>726</td>\n      <td>370</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2010-12-24 03:17:41</td>\n      <td>0</td>\n      <td>@jbirdbradyII My freshman year of college I wasn’t allowed to enroll in classes because my 2nd MMR vaccine was given when I was 360 days old and not 1 year. So at 18 I got a baby vaccine. Also @drdrew is an intentional idiot. He just needed some attention today. Here it is sweetie.</td>\n      <td>[]</td>\n      <td>[@jbirdbradyII, @drdrew]</td>\n      <td>freshman year college wasnt allow enroll class mmr vaccine give day old year get baby vaccine also intentional idiot need attention today sweetie</td>\n      <td>Negative</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>BettyBunny23</td>\n      <td>clotted cream first.</td>\n      <td>2866</td>\n      <td>2674</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2015-06-01 20:54:44</td>\n      <td>3</td>\n      <td>@Nigel_Farage I am slightly confused we we were told the vaccine was the be all and end all !! Life back to normal so why are they bringing in the vaccine passport when 90% of the country would’ve been vaccinated ? It sounds like they were not quite as confident as they were .</td>\n      <td>[]</td>\n      <td>[@Nigel_Farage]</td>\n      <td>slightly confuse tell vaccine end life back normal bringing vaccine passport country wouldve vaccinated sound like quite confident</td>\n      <td>Positive</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>dwm2112</td>\n      <td>galt's gulch</td>\n      <td>350</td>\n      <td>163</td>\n      <td>2021-04-05 20:28:20</td>\n      <td>2009-04-28 19:44:48</td>\n      <td>5902</td>\n      <td>Requiring Americans to show proof of vaccination, or a ‘vaccine passport,’ in order to engage in society is outrageous and un-American. This isn’t about containing a virus, but is just another dangerous idea from the Left in their quest for more power and control.</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>require american show proof vaccination vaccine passport, order engage society outrageous un-american isnt contain virus another dangerous idea leave quest power control</td>\n      <td>Negative</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 56
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['sentiment'].replace(to_replace = 1, value = 'Positive', inplace = True)\n",
    "data['sentiment'].replace(to_replace = -1, value = 'Negative', inplace = True)\n",
    "data['sentiment'].replace(to_replace = 0, value = 'Neutral', inplace = True)"
   ]
  },
  {
   "source": [
    "data.to_csv('./datasets/cleaned/vaccines-labeled-lexicon.csv', index = False)"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 57,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python386jvsc74a57bd07c24553cd23581219c3e61884a4dae72365ef7f0d9c5c27a3959e30dfcb4b767",
   "display_name": "Python 3.8.6rc1 64-bit"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "metadata": {
   "interpreter": {
    "hash": "7c24553cd23581219c3e61884a4dae72365ef7f0d9c5c27a3959e30dfcb4b767"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}