Social Web Latest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     C:\\Users\\LPras\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import glob\n",
    "import pandas as pd\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "nltk.download('vader_lexicon')\n",
    "sid = SentimentIntensityAnalyzer()\n",
    "pd.options.display.max_colwidth = 10000\n",
    "#2020-11-02-01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vader on our dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hydrated_tweets/2020-11\n",
      "['hydrated_tweets/2020-11\\\\df11-01-00.csv', 'hydrated_tweets/2020-11\\\\df11-01-01.csv', 'hydrated_tweets/2020-11\\\\df11-01-02.csv', 'hydrated_tweets/2020-11\\\\df11-01-03.csv', 'hydrated_tweets/2020-11\\\\df11-01-04.csv', 'hydrated_tweets/2020-11\\\\df11-01-05.csv', 'hydrated_tweets/2020-11\\\\df11-01-06.csv', 'hydrated_tweets/2020-11\\\\df11-01-07.csv', 'hydrated_tweets/2020-11\\\\df11-01-08.csv', 'hydrated_tweets/2020-11\\\\df11-01-09.csv', 'hydrated_tweets/2020-11\\\\df11-01-10.csv', 'hydrated_tweets/2020-11\\\\df11-01-11.csv', 'hydrated_tweets/2020-11\\\\df11-01-12.csv', 'hydrated_tweets/2020-11\\\\df11-01-13.csv', 'hydrated_tweets/2020-11\\\\df11-01-14.csv', 'hydrated_tweets/2020-11\\\\df11-01-15.csv', 'hydrated_tweets/2020-11\\\\df11-01-16.csv', 'hydrated_tweets/2020-11\\\\df11-01-17.csv', 'hydrated_tweets/2020-11\\\\df11-01-18.csv', 'hydrated_tweets/2020-11\\\\df11-01-19.csv', 'hydrated_tweets/2020-11\\\\df11-01-20.csv', 'hydrated_tweets/2020-11\\\\df11-01-21.csv', 'hydrated_tweets/2020-11\\\\df11-01-22.csv', 'hydrated_tweets/2020-11\\\\df11-01-23.csv', 'hydrated_tweets/2020-11\\\\df11-02-00.csv', 'hydrated_tweets/2020-11\\\\df11-02-01.csv', 'hydrated_tweets/2020-11\\\\df11-02-02.csv', 'hydrated_tweets/2020-11\\\\df11-02-03.csv', 'hydrated_tweets/2020-11\\\\df11-02-04.csv', 'hydrated_tweets/2020-11\\\\df11-02-05.csv', 'hydrated_tweets/2020-11\\\\df11-02-06.csv', 'hydrated_tweets/2020-11\\\\df11-02-07.csv', 'hydrated_tweets/2020-11\\\\df11-02-08.csv', 'hydrated_tweets/2020-11\\\\df11-02-09.csv', 'hydrated_tweets/2020-11\\\\df11-02-10.csv', 'hydrated_tweets/2020-11\\\\df11-02-11.csv', 'hydrated_tweets/2020-11\\\\df11-02-12.csv', 'hydrated_tweets/2020-11\\\\df11-02-13.csv', 'hydrated_tweets/2020-11\\\\df11-02-14.csv', 'hydrated_tweets/2020-11\\\\df11-02-15.csv', 'hydrated_tweets/2020-11\\\\df11-02-16.csv', 'hydrated_tweets/2020-11\\\\df11-02-17.csv', 'hydrated_tweets/2020-11\\\\df11-02-18.csv', 'hydrated_tweets/2020-11\\\\df11-02-19.csv', 'hydrated_tweets/2020-11\\\\df11-02-20.csv', 'hydrated_tweets/2020-11\\\\df11-02-21.csv', 'hydrated_tweets/2020-11\\\\df11-02-22.csv', 'hydrated_tweets/2020-11\\\\df11-02-23.csv', 'hydrated_tweets/2020-11\\\\df11-03-00.csv', 'hydrated_tweets/2020-11\\\\df11-03-01.csv', 'hydrated_tweets/2020-11\\\\df11-03-02.csv', 'hydrated_tweets/2020-11\\\\df11-03-03.csv', 'hydrated_tweets/2020-11\\\\df11-03-04.csv', 'hydrated_tweets/2020-11\\\\df11-03-05.csv', 'hydrated_tweets/2020-11\\\\df11-03-06.csv', 'hydrated_tweets/2020-11\\\\df11-03-07.csv', 'hydrated_tweets/2020-11\\\\df11-03-08.csv', 'hydrated_tweets/2020-11\\\\df11-03-09.csv', 'hydrated_tweets/2020-11\\\\df11-03-10.csv', 'hydrated_tweets/2020-11\\\\df11-03-11.csv', 'hydrated_tweets/2020-11\\\\df11-03-12.csv', 'hydrated_tweets/2020-11\\\\df11-03-13.csv', 'hydrated_tweets/2020-11\\\\df11-03-14.csv', 'hydrated_tweets/2020-11\\\\df11-03-15.csv', 'hydrated_tweets/2020-11\\\\df11-03-16.csv', 'hydrated_tweets/2020-11\\\\df11-03-17.csv', 'hydrated_tweets/2020-11\\\\df11-03-18.csv', 'hydrated_tweets/2020-11\\\\df11-03-19.csv', 'hydrated_tweets/2020-11\\\\df11-03-20.csv', 'hydrated_tweets/2020-11\\\\df11-03-21.csv', 'hydrated_tweets/2020-11\\\\df11-03-22.csv', 'hydrated_tweets/2020-11\\\\df11-03-23.csv', 'hydrated_tweets/2020-11\\\\df11-04-00.csv', 'hydrated_tweets/2020-11\\\\df11-04-01.csv', 'hydrated_tweets/2020-11\\\\df11-04-02.csv', 'hydrated_tweets/2020-11\\\\df11-04-03.csv', 'hydrated_tweets/2020-11\\\\df11-04-04.csv', 'hydrated_tweets/2020-11\\\\df11-04-05.csv', 'hydrated_tweets/2020-11\\\\df11-04-06.csv', 'hydrated_tweets/2020-11\\\\df11-04-07.csv', 'hydrated_tweets/2020-11\\\\df11-04-08.csv', 'hydrated_tweets/2020-11\\\\df11-04-09.csv', 'hydrated_tweets/2020-11\\\\df11-04-10.csv', 'hydrated_tweets/2020-11\\\\df11-04-11.csv', 'hydrated_tweets/2020-11\\\\df11-04-12.csv', 'hydrated_tweets/2020-11\\\\df11-04-13.csv', 'hydrated_tweets/2020-11\\\\df11-04-14.csv', 'hydrated_tweets/2020-11\\\\df11-04-15.csv', 'hydrated_tweets/2020-11\\\\df11-04-16.csv', 'hydrated_tweets/2020-11\\\\df11-04-17.csv', 'hydrated_tweets/2020-11\\\\df11-04-18.csv', 'hydrated_tweets/2020-11\\\\df11-04-19.csv', 'hydrated_tweets/2020-11\\\\df11-04-20.csv', 'hydrated_tweets/2020-11\\\\df11-04-21.csv', 'hydrated_tweets/2020-11\\\\df11-04-22.csv', 'hydrated_tweets/2020-11\\\\df11-04-23.csv']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\LPras\\AppData\\Local\\Temp\\ipykernel_21920\\159643915.py:6: DtypeWarning: Columns (0,5,6,7,8,11) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.concat((pd.read_csv(f, sep=';') for f in all_files), ignore_index=True)\n"
     ]
    },
    {
     "ename": "ParserError",
     "evalue": "Error tokenizing data. C error: Expected 12 fields in line 474, saw 13\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mParserError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_21920\\159643915.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mall_files\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mglob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mglob\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"*.csv\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mall_files\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m';'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mall_files\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msubset\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'location'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'all'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\core\\reshape\\concat.py\u001b[0m in \u001b[0;36mconcat\u001b[1;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[0;32m    272\u001b[0m     \u001b[0mValueError\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mIndexes\u001b[0m \u001b[0mhave\u001b[0m \u001b[0moverlapping\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'a'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    273\u001b[0m     \"\"\"\n\u001b[1;32m--> 274\u001b[1;33m     op = _Concatenator(\n\u001b[0m\u001b[0;32m    275\u001b[0m         \u001b[0mobjs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    276\u001b[0m         \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\core\\reshape\\concat.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001b[0m\n\u001b[0;32m    326\u001b[0m             \u001b[0mobjs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mobjs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mkeys\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    327\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 328\u001b[1;33m             \u001b[0mobjs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobjs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    329\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    330\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobjs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_21920\\159643915.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mall_files\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mglob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mglob\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"*.csv\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mall_files\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m';'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mall_files\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msubset\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'location'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'all'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[0;32m    684\u001b[0m     )\n\u001b[0;32m    685\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 686\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    687\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    688\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    456\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    457\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 458\u001b[1;33m         \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    459\u001b[0m     \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    460\u001b[0m         \u001b[0mparser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, nrows)\u001b[0m\n\u001b[0;32m   1194\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1195\u001b[0m         \u001b[0mnrows\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_validate_integer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"nrows\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1196\u001b[1;33m         \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1197\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1198\u001b[0m         \u001b[1;31m# May alter columns / col_dict\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\miniconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, nrows)\u001b[0m\n\u001b[0;32m   2153\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2154\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2155\u001b[1;33m             \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2156\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2157\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_first_chunk\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.read\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_low_memory\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._tokenize_rows\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.raise_parser_error\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mParserError\u001b[0m: Error tokenizing data. C error: Expected 12 fields in line 474, saw 13\n"
     ]
    }
   ],
   "source": [
    "# Change path to directory with files\n",
    "path = 'hydrated_tweets/2020-11'\n",
    "all_files = glob.glob(os.path.join(path, \"*.csv\"))\n",
    "df = pd.concat((pd.read_csv(f, sep=';') for f in all_files), ignore_index=True)\n",
    "df = df.dropna(subset=['location'], how='all')\n",
    "df['text'] = df['text'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "678935"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "states = ['Alabama', 'Alaska', 'Arizona', 'Arkansas', 'California', 'Colorado', 'Connecticut', 'Delaware', 'Florida', 'Georgia', 'Hawaii', 'Idaho', 'Illinois', 'Indiana', 'Iowa', 'Kansas', 'Kentucky', 'Louisiana', 'Maine', 'Maryland', 'Massachusetts', 'Michigan', 'Minnesota', 'Mississippi', 'Missouri', 'Montana', 'Nebraska', 'Nevada', 'New Hampshire', 'New Jersey', 'New York', 'New Mexico', 'North Carolina', 'North Dakota', 'Ohio', 'Oklahoma', 'Oregon', 'Pennsylvania', 'Rhode Island', 'South Carolina', 'South Dakota', 'Tennessee', 'Texas', 'Utah', 'Vermont', 'Virginia', 'Washington', 'West Virginia', 'Wisconsin', 'Wyoming']\n",
    "stateCodes = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA', 'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD', 'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY']\n",
    "stateMapping = {'AL': 'Alabama', 'AK': 'Alaska', 'AZ': 'Arizona', 'AR': 'Arkansas', 'CA': 'California', 'CO': 'Colorado', 'CT': 'Connecticut', 'DE': 'Delaware', 'FL': 'Florida', 'GA': 'Georgia', 'HI': 'Hawaii', 'ID': 'Idaho', 'IL': 'Illinois', 'IN': 'Indiana', 'IA': 'Iowa', 'KS': 'Kansas', 'KY': 'Kentucky', 'LA': 'Louisiana', 'ME': 'Maine', 'MD': 'Maryland', 'MA': 'Massachusetts', 'MI': 'Michigan', 'MN': 'Minnesota', 'MS': 'Mississippi', 'MO': 'Missouri', 'MT': 'Montana', 'NE': 'Nebraska', 'NV': 'Nevada', 'NH': 'New Hampshire', 'NJ': 'New Jersey', 'NY': 'New York', 'NM': 'New Mexico', 'NC': 'North Carolina', 'ND': 'North Dakota', 'OH': 'Ohio', 'OK': 'Oklahoma', 'OR': 'Oregon', 'PA': 'Pennsylvania', 'RI': 'Rhode Island', 'SC': 'South Carolina', 'SD': 'South Dakota', 'TN': 'Tennessee', 'TX': 'Texas', 'UT':  'Utah', 'VT': 'Vermont', 'VA': 'Virginia', 'WA': 'Washington', 'WV':  'West Virginia', 'WI': 'Wisconsin', 'WY': 'Wyoming'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_terms = pd.read_csv('search_terms.csv', sep=';')\n",
    "df_terms['term'] = df_terms ['term'].str.lower()\n",
    "#df_terms[(df_terms['party'].str.contains('N'))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check terms\n",
    "df_trump = df[(df['text'].str.contains('trump')) | (df['text'].str.contains('maga')) | (df['text'].str.contains('fraud'))]\n",
    "df_biden = df[(df['text'].str.contains('biden')) | (df['text'].str.contains('kamala'))| (df['text'].str.contains('harris'))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check state locations\n",
    "states_dict_trump = {}\n",
    "for state in stateCodes:\n",
    "    states_dict_trump[stateMapping[state]] = df_trump[df_trump['location'].str.contains(state+ '|' + stateMapping[state])]\n",
    "    \n",
    "states_dict_biden = {}\n",
    "for state in stateCodes:\n",
    "    states_dict_biden[stateMapping[state]] = df_biden[df_biden['location'].str.contains(state+ '|' + stateMapping[state])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "103322"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count = 0\n",
    "for state in states:\n",
    "    count+= len(states_dict_biden[state])\n",
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "209130"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count = 0\n",
    "for state in states:\n",
    "    count+= len(states_dict_trump[state])\n",
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "678935"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-11-49a308d8266e>:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  states_dict_biden[state]['sentiment_scores'] = states_dict_biden[state]['text'].apply(lambda x: sid.polarity_scores(x))\n",
      "<ipython-input-11-49a308d8266e>:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  states_dict_biden[state]['sentiment'] = states_dict_biden[state]['sentiment_scores'].apply(lambda x: sentimentVerdict(x))\n",
      "<ipython-input-11-49a308d8266e>:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  states_dict_trump[state]['sentiment_scores'] = states_dict_trump[state]['text'].apply(lambda x: sid.polarity_scores(x))\n",
      "<ipython-input-11-49a308d8266e>:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  states_dict_trump[state]['sentiment'] = states_dict_trump[state]['sentiment_scores'].apply(lambda x: sentimentVerdict(x))\n"
     ]
    }
   ],
   "source": [
    "# Run the sentiment analysis on each state for trump and biden\n",
    "def sentimentVerdict(sentiment):\n",
    "    if sentiment['compound'] >= 0.05:\n",
    "        return \"Positive\"\n",
    "    elif sentiment['compound'] <= -0.05:\n",
    "        return \"Negative\"\n",
    "    else:\n",
    "        return \"Neutral\"\n",
    "    \n",
    "for state in states:\n",
    "    states_dict_biden[state]['sentiment_scores'] = states_dict_biden[state]['text'].apply(lambda x: sid.polarity_scores(x))\n",
    "    states_dict_biden[state]['sentiment'] = states_dict_biden[state]['sentiment_scores'].apply(lambda x: sentimentVerdict(x))\n",
    "    states_dict_trump[state]['sentiment_scores'] = states_dict_trump[state]['text'].apply(lambda x: sid.polarity_scores(x))\n",
    "    states_dict_trump[state]['sentiment'] = states_dict_trump[state]['sentiment_scores'].apply(lambda x: sentimentVerdict(x)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Election data, don´t have to use this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Election data\n",
    "df_elec = pd.read_csv('election_data.csv', sep=',')\n",
    "df_elec = df_elec[['state', 'candidate', 'total_votes', 'won']]\n",
    "df_elec = df_elec[((df_elec.candidate == 'Joe Biden') | (df_elec.candidate == 'Donald Trump')) & (df_elec.state != 'District of Columbia')]\n",
    "df_elec = df_elec.groupby(['state', 'candidate'])['total_votes'].apply(sum).reset_index()\n",
    "\n",
    "percentagelist = []\n",
    "for state in states:\n",
    "    bidentot = df_elec.loc[(df_elec['state'] == state) & (df_elec['candidate'] == 'Joe Biden')]['total_votes']\n",
    "    trumptot = df_elec.loc[(df_elec['state'] == state) & (df_elec['candidate'] == 'Donald Trump')]['total_votes']\n",
    "    bidenpercentage = int(bidentot) / (int(bidentot) + int(trumptot))\n",
    "    trumppercentage = int(trumptot) / (int(bidentot) + int(trumptot))\n",
    "    percentagelist.append(trumppercentage)\n",
    "    percentagelist.append(bidenpercentage)\n",
    "    \n",
    "df_elec['ratio of votes'] = percentagelist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment % pos neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Percentage sentiment per state\n",
    "state_sentiment_trump = {}\n",
    "state_sentiment_biden = {}\n",
    "for state in states:\n",
    "    # Trump\n",
    "    d_t = dict(states_dict_trump[state]['sentiment'].value_counts())\n",
    "    tot = d_t['Positive']+d_t['Negative']# +d_t['Neutral']\n",
    "    post= d_t['Positive'] / tot\n",
    "    negt= d_t['Negative'] / tot\n",
    "    # Biden\n",
    "    d_b = dict(states_dict_biden[state]['sentiment'].value_counts())\n",
    "    tot = d_b['Positive']+d_b['Negative']# +d_b['Neutral']\n",
    "    posb= d_b['Positive'] / tot\n",
    "    negb= d_b['Negative'] / tot\n",
    "       \n",
    "    state_sentiment_trump[state] = {'positive' : post, 'negative': negt}\n",
    "    state_sentiment_biden[state] = {'positive' : posb, 'negative': negb}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# put sentiments and election outcomes in list for plotting\n",
    "positive_sentiments_states_trump = []\n",
    "elec_result_states_trump = []\n",
    "positive_sentiments_states_biden = []\n",
    "elec_result_states_biden = []\n",
    "for state in states:\n",
    "    positive_sentiments_states_trump.append(state_sentiment_trump[state]['positive'])\n",
    "    elec_result_states_trump.append(list(df_elec.loc[(df_elec['state'] == state) & (df_elec['candidate'] == 'Donald Trump')]['ratio of votes'])[0])\n",
    "    positive_sentiments_states_biden.append(state_sentiment_biden[state]['positive'])\n",
    "    elec_result_states_biden.append(list(df_elec.loc[(df_elec['state'] == state) & (df_elec['candidate'] == 'Joe Biden')]['ratio of votes'])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# put sentiments and election outcomes in list for plotting\n",
    "negative_sentiments_states_trump = []\n",
    "elec_result_states_trump = []\n",
    "negative_sentiments_states_biden = []\n",
    "elec_result_states_biden = []\n",
    "for state in states:\n",
    "    negative_sentiments_states_trump.append(state_sentiment_trump[state]['negative'])\n",
    "    elec_result_states_trump.append(list(df_elec.loc[(df_elec['state'] == state) & (df_elec['candidate'] == 'Donald Trump')]['ratio of votes'])[0])\n",
    "    negative_sentiments_states_biden.append(state_sentiment_biden[state]['negative'])\n",
    "    elec_result_states_biden.append(list(df_elec.loc[(df_elec['state'] == state) & (df_elec['candidate'] == 'Joe Biden')]['ratio of votes'])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for state in states:\n",
    "    print(state)\n",
    "    print(\"trump\")\n",
    "\n",
    "    print(state_sentiment_trump[state])\n",
    "    print(\"Biden\")\n",
    "\n",
    "    print(state_sentiment_biden[state])\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_sentiment_trump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_sentiment_biden"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Current plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(states, positive_sentiments_states_biden, alpha = 0.5, label=\"Positive sentiment\")\n",
    "plt.bar(states, elec_result_states_biden, alpha = 0.5, label = 'Percentage of votes')\n",
    "plt.xticks(rotation=90)\n",
    "plt.ylabel(\"Percentage positive/votes\")\n",
    "plt.xlabel(\"States\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(states, positive_sentiments_states_trump, alpha = 0.5, label=\"Positive sentiment\")\n",
    "plt.bar(states, elec_result_states_trump, alpha = 0.5, label = 'Percentage of votes')\n",
    "plt.xticks(rotation=90)\n",
    "plt.ylabel(\"Percentage positive/votes\")\n",
    "plt.xlabel(\"States\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}