LaManada.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style='text-align: center;'>\n",
    "    <h1>#LaManada</h1>\n",
    "    <h2>Mini-Project CSS Summer School</h2>\n",
    "    <div>July 30th - August 4th, 2018</div>\n",
    "    <div>Marina Del Rey, C.A.</div>\n",
    "</div>\n",
    "\n",
    "<p>\n",
    "    <b>Members</b>\n",
    "    <ul>\n",
    "        <li>Blanca Ramirez, USC (USA)</li>\n",
    "        <li>Tayrine Dias, UOC (Spain)</li>\n",
    "        <li>Lisette Espin-Noboa, GESIS (Germany)</li>\n",
    "    </ul>\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Descriptive Analysis</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################\n",
    "# GENERAL DEPENDENCES\n",
    "####################################################\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from wordcloud import WordCloud\n",
    "import multidict as multidict\n",
    "import re\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################\n",
    "# NLP DEPENDENCES\n",
    "####################################################\n",
    "import nltk  \n",
    "from nltk.corpus import stopwords  \n",
    "from nltk import word_tokenize  \n",
    "from nltk.data import load  \n",
    "from nltk.stem import SnowballStemmer  \n",
    "from string import punctuation  \n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "#stopword list to use\n",
    "spanish_stopwords = stopwords.words('spanish')\n",
    "my_stopwords = ['me', 'las', 'es', 'un', 'mi', 'con', 'ser', 'los', 'si', 'ha', 'hasta', 'o', 'de', 'cuando', 'http', 'su', 'twitter', 'er', 'como', 'to', 'le', 'se', 'en', 'lo', 'a', 'tgo', 'toy', 'tu', 'el', 'por', 'una', 'al', 'para', 'la', 'pero', 'que', 'da', 'https', 'y','q','del','xq','les','mis','te','sí','ya','i','porque','por que','por qué','era','cada','nos','pero','ni']\n",
    "with open('stopwords_ca.txt','r') as f:\n",
    "    catalan_stopwords=[line.replace('\\n','') for line in f.readlines()]\n",
    "all_stopwords = set(my_stopwords)\n",
    "all_stopwords |= set(spanish_stopwords)\n",
    "all_stopwords |= set(catalan_stopwords)\n",
    "\n",
    "#spanish stemmer\n",
    "stemmer = SnowballStemmer('spanish')\n",
    "\n",
    "#punctuation to remove\n",
    "non_words = list(punctuation)\n",
    "#we add spanish punctuation\n",
    "non_words.extend(['¿', '¡','\"',\"'\",\"“\",\"”\",\"‘\",\"’\",\"$\",\"€\",\"<\",\">\",\"^\",\"`\",\"~\",\"«\",\"»\"])  \n",
    "non_words.extend(map(str,range(10)))\n",
    "\n",
    "stemmer = SnowballStemmer('spanish')  \n",
    "def stem_tokens(tokens, stemmer):  \n",
    "    stemmed = []\n",
    "    for item in tokens:\n",
    "        if item not in all_stopwords:\n",
    "            stemmed.append(stemmer.stem(item))\n",
    "    return stemmed\n",
    "\n",
    "def tokenize(text):  \n",
    "    # remove punctuation\n",
    "    text = ''.join([c for c in text if c not in non_words])\n",
    "    # tokenize\n",
    "    tokens =  word_tokenize(text)\n",
    "\n",
    "    # stem\n",
    "    try:\n",
    "        stems = stem_tokens(tokens, stemmer)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(text)\n",
    "        stems = ['']\n",
    "    return stems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################\n",
    "# CONSTANTS\n",
    "####################################################\n",
    "K = 10\n",
    "SEP = ','"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################\n",
    "# FUNCTIONS (HANDLERS)\n",
    "####################################################\n",
    "def getFrequencyDictForText(sentence):\n",
    "    fullTermsDict = multidict.MultiDict()\n",
    "    tmpDict = {}\n",
    "    # making dict for counting frequencies\n",
    "    for text in sentence.split(\" \"):\n",
    "        val = tmpDict.get(text, 0)\n",
    "        tmpDict[text.lower()] = val + 1\n",
    "    for key in tmpDict:\n",
    "        fullTermsDict.add(key, tmpDict[key])\n",
    "    return {k:v for k,v in fullTermsDict.items() if v>1}\n",
    "\n",
    "def makeImage(text, width, height, shape):\n",
    "    wc = WordCloud(background_color=\"white\", max_words=2000, width=width, height=height, stopwords=all_stopwords)\n",
    "    wc.generate_from_frequencies(text)\n",
    "    plt.figure(figsize=shape)\n",
    "    plt.imshow(wc, interpolation=\"bilinear\")\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "\n",
    "def _get_text(values):\n",
    "    return ' '.join([_get_tokens(t) for t in values])\n",
    "    \n",
    "def _get_tokens(text):\n",
    "    text = re.sub(r\"http\\S+\", \"\", text)\n",
    "    text = re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', text, flags=re.MULTILINE)\n",
    "    text = re.sub(r'^http?:\\/\\/.*[\\r\\n]*', '', text, flags=re.MULTILINE)\n",
    "    text = ''.join([c for c in text if c not in non_words])\n",
    "    tokens =  word_tokenize(text)\n",
    "    text = []\n",
    "    for item in tokens:\n",
    "        if item not in all_stopwords:\n",
    "            text.append(item)\n",
    "    text = ' '.join(text)\n",
    "    return text\n",
    "    \n",
    "def remove_stop_words_es(text):\n",
    "    text = text.lower()\n",
    "    for word in all_stopwords:\n",
    "        text = text.replace(' {} '.format(word),'')\n",
    "        text = text.replace('{} '.format(word),'') if text.startswith('{} '.format(word)) else text\n",
    "        text = text.replace(' {}'.format(word),'') if text.endswith(' {}'.format(word)) else text\n",
    "        \n",
    "    text = text.replace('?','').replace('.','').replace(',','').replace(':','').replace(';','').replace('\"','').replace(\"'\",\"\")\n",
    "    return text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################\n",
    "# LOADING THE DATA\n",
    "####################################################\n",
    "users = pd.read_csv('LaManada_new/tbluserinfo.csv',sep=SEP)\n",
    "users.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = pd.read_csv('LaManada_new/tblposts.csv',sep=SEP)\n",
    "tweets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "retweets = pd.read_csv('LaManada_new/tblretweets.csv',sep=SEP)\n",
    "retweets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "replies = pd.read_csv('LaManada_new/tblreplies.csv',sep=SEP,quotechar='\"')\n",
    "replies.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Columns</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "users.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "retweets.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "replies.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Who are tweeting about #LaManada?</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "users[\"created_at\"] = users[\"created_at\"].astype(\"datetime64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "users.query(\"created_at < '1984-01-01 00:00:00'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = users.query(\"created_at > '1984-01-01 00:00:00'\") #removing those 2 outliers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Accounts creation date</h3>\n",
    "There is a peak of ~4000 users that were created in 2018 (probably trolls?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = users.created_at.hist(bins=12)\n",
    "ax.set_xlabel('Date')\n",
    "ax.set_ylabel('# Users')\n",
    "ax.set_title(\"Users' account creation per year\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = users.created_at.hist(bins=144)\n",
    "ax.set_xlabel('Date')\n",
    "ax.set_ylabel('# Users')\n",
    "ax.set_title(\"Users' account creation per month\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = users.created_at.hist(bins=4380)\n",
    "ax.set_xlabel('Date')\n",
    "ax.set_ylabel('# Users')\n",
    "ax.set_title(\"Users' account creation per day\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 27-April - 4 May\n",
    "users.loc[:,'creation_date'] = users.created_at.dt.date\n",
    "tmp = users.groupby(['creation_date']).size().reset_index()\n",
    "tmp.rename(columns={0:'users'},inplace=True)\n",
    "ax=sns.scatterplot(data=tmp, x='creation_date', y='users')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>User's # Favorites (in general)</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = users.favourites_count.hist(bins=500)\n",
    "ax.set_yscale('log')\n",
    "ax.set_xscale('log')\n",
    "ax.set_xlabel('Favorite Counts')\n",
    "ax.set_ylabel('# Users')\n",
    "ax.set_title(\"Users' Favorite counts\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>User's listed count</h3>\n",
    "Around 1M users have been listed in $\\leq$ 100 lists. <br />\n",
    "Only a few users (~10) have been listed in at least 2K lists."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = users.listed_count.hist(bins=500)\n",
    "ax.set_yscale('log')\n",
    "ax.set_xscale('log')\n",
    "ax.set_xlabel('Lists Counts')\n",
    "ax.set_ylabel('# Users')\n",
    "ax.set_title(\"Users' Lists counts\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>#Friends vs #Followers</h3>\n",
    "Verified accounts (right columns) tend to have more followers and a few friends compared to non-verified accounts <br />\n",
    "This makes sense, since verified users are usually famous people such as celebrities, politicians, etc. Therefore, they have more followers than 'normal' users. <br />\n",
    "There is no significant different between users with geo_enabled 0 or 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.FacetGrid(users, col=\"verified\",  row=\"geo_enabled\", margin_titles=True)\n",
    "g = g.map(sns.regplot, 'followers_count', 'friends_count', fit_reg=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Most Active Users</h2>\n",
    "These are the top 10 users who posted/retweeted the most. <br />\n",
    "To be an active user, one must have tweeted at least 5 different posts.<br />\n",
    "None of them are verified accounts."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>All Posts</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Most active users posting new tweets and retweeting\n",
    "# removing same-text tweet posted multiple times per user.\n",
    "uniquetweets = tweets.groupby(['snsuserid']).text.nunique().reset_index() \n",
    "uniquetweets.rename(columns={'text':'nu_tweets'},inplace=True)\n",
    "useractivity = uniquetweets.query(\"nu_tweets>5\") #112448\n",
    "useractivity.reset_index()\n",
    "useractivity.sort_values('nu_tweets', ascending=False, inplace=True)\n",
    "useractivitytopk_all = useractivity.iloc[:K,:]\n",
    "for i,row in useractivitytopk_all.iterrows():\n",
    "    useractivitytopk_all.loc[i,'name'] = users.query(\"snsuserid==@row.snsuserid\")['screen_name'].values[0]\n",
    "    useractivitytopk_all.loc[i,'verified'] = users.query(\"snsuserid==@row.snsuserid\")['verified'].values[0]\n",
    "ax = sns.barplot(data=useractivitytopk_all, y='nu_tweets', x='name')\n",
    "ax.set_yscale('log')\n",
    "ax.set_ylabel('# All Posts')\n",
    "ax.set_xlabel('Top{} Active Users'.format(K))\n",
    "ax.set_title('Most Active Users')\n",
    "ax.xaxis.set_tick_params(rotation=90)\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>What are they talking about?</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = tweets.query(\"snsuserid.isin(@useractivitytopk_all.snsuserid.values)\")\n",
    "text = _get_text(tmp.text.values)\n",
    "makeImage(getFrequencyDictForText(text), width=400, height=400, shape=(6,6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Only New Tweets</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Most active users posting new tweets\n",
    "uniquetweets = tweets.query(\"isaRetweet==0\").groupby(['snsuserid']).text.nunique().reset_index() \n",
    "uniquetweets.rename(columns={'text':'nu_tweets'},inplace=True)\n",
    "useractivity = uniquetweets.query(\"nu_tweets>5\") #112448\n",
    "useractivity.reset_index()\n",
    "useractivity.sort_values('nu_tweets', ascending=False, inplace=True)\n",
    "useractivitytopk_tweets = useractivity.iloc[:K,:]\n",
    "for i,row in useractivitytopk_tweets.iterrows():\n",
    "    useractivitytopk_tweets.loc[i,'name'] = users.query(\"snsuserid==@row.snsuserid\")['screen_name'].values[0]\n",
    "    useractivitytopk_tweets.loc[i,'verified'] = users.query(\"snsuserid==@row.snsuserid\")['verified'].values[0]\n",
    "ax = sns.barplot(data=useractivitytopk_tweets, y='nu_tweets', x='name')\n",
    "ax.set_yscale('log')\n",
    "ax.set_ylabel('# New TweetsAll')\n",
    "ax.set_xlabel('Top{} Active Users'.format(K))\n",
    "ax.set_title('Most Active Users')\n",
    "ax.xaxis.set_tick_params(rotation=90)\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>What are they talking about?</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = tweets.query(\"snsuserid.isin(@useractivitytopk_tweets.snsuserid.values)\")\n",
    "text = _get_text(tmp.text.values)\n",
    "makeImage(getFrequencyDictForText(text), width=400, height=400, shape=(6,6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Spammers</h2>\n",
    "These are those users who posted the same tweet multiple times."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "possible_bots = users.query(\"created_at > '2018-04-26'\")\n",
    "possible_bots.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = possible_bots.groupby(['followers_count']).size().reset_index()\n",
    "tmp.rename(columns={0:'nusers'},inplace=True)\n",
    "ax = sns.scatterplot(data=tmp,y='nusers',x='followers_count')\n",
    "ax.set_yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_possible_bots = tweets.query(\"snsuserid.isin(@possible_bots.snsuserid.values)\")\n",
    "tweets_possible_bots.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_possible_bots.query(\"isaRetweet==1\").shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,r in tweets_possible_bots.query(\"isaRetweet==1\").sort_values(by='numRetweets', ascending=False).iloc[:10].iterrows():\n",
    "    print(r.numRetweets)\n",
    "    print(r.numFavourites)\n",
    "    print(r.snsuserid)\n",
    "    print(users.query(\"snsuserid == @r.snsuserid\").values)\n",
    "    print(r.snspostid)\n",
    "    print(r.text)\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets.query(\"text.str.contains('repúblicadominicana',False)\").shape\n",
    "# for i,r in tweets.query(\"text.str.contains('chile',False)\").iterrows():\n",
    "#     print(r.text)\n",
    "#     print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# by repeated tweet\n",
    "tmp = tweets.groupby(['snsuserid','text']).size().reset_index()\n",
    "tmp.rename(columns={0:'counts'},inplace=True)\n",
    "tmp.sort_values(by=['counts'],ascending=False).query(\"counts > 2\").counts.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Tweets</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets[\"created_at\"] = tweets[\"created_at\"].astype(\"datetime64\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Tweets Activity</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = tweets.created_at.hist(bins=8)\n",
    "ax.set_xlabel('Date')\n",
    "ax.set_ylabel('# Tweets')\n",
    "ax.set_title(\"Activity per Day\")\n",
    "ax.xaxis.set_tick_params(rotation=90)\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ax = tweets.created_at.hist(bins=192)\n",
    "ax.set_xlabel('Date')\n",
    "ax.set_ylabel('# Tweets')\n",
    "ax.set_title(\"Activity per Hour\")\n",
    "ax.xaxis.set_tick_params(rotation=90)\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets['text_fmt'] = tweets['text'].apply(lambda x:_get_tokens(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>All Tweets</h3>\n",
    "(tweets and retweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = ' '.join(tweets.text_fmt.values)\n",
    "wc = WordCloud(background_color=\"white\", max_words=5000, width=2000, height=1000, stopwords=all_stopwords)\n",
    "wc.generate_from_frequencies(getFrequencyDictForText(text))\n",
    "plt.figure(figsize=(40,20))\n",
    "plt.imshow(wc, interpolation=\"bilinear\")\n",
    "plt.xticks([])\n",
    "plt.yticks([])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Tweet-Only Text</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = ' '.join(tweets.query(\"isaRetweet==0\").text_fmt.values)\n",
    "wc = WordCloud(background_color=\"white\", max_words=5000, width=2000, height=1000, stopwords=all_stopwords)\n",
    "wc.generate_from_frequencies(getFrequencyDictForText(text))\n",
    "plt.figure(figsize=(40,20))\n",
    "plt.imshow(wc, interpolation=\"bilinear\")\n",
    "plt.xticks([])\n",
    "plt.yticks([])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Retweets-Only Text</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = ' '.join(tweets.query(\"isaRetweet==1\").text_fmt.values)\n",
    "wc = WordCloud(background_color=\"white\", max_words=5000, width=2000, height=1000, stopwords=all_stopwords)\n",
    "wc.generate_from_frequencies(getFrequencyDictForText(text))\n",
    "plt.figure(figsize=(40,20))\n",
    "plt.imshow(wc, interpolation=\"bilinear\")\n",
    "plt.xticks([])\n",
    "plt.yticks([])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Topic Modeling (NMF)</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets['text_fmt_stem'] = tweets['text'].apply(lambda x:stemmer.stem(_get_tokens(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = tweets.text_fmt.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#http://scikit-learn.org/0.18/auto_examples/applications/topics_extraction_with_nmf_lda.html\n",
    "from __future__ import print_function\n",
    "from time import time\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.decomposition import NMF, LatentDirichletAllocation\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "n_samples = documents.shape[0]\n",
    "n_features = 1000\n",
    "n_topics = 5\n",
    "n_top_words = 20\n",
    "def print_top_words(model, feature_names, n_top_words):\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        print(\"Topic #%d:\" % topic_idx)\n",
    "        print(\" \".join([feature_names[i]\n",
    "                        for i in topic.argsort()[:-n_top_words - 1:-1]]))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use tf-idf features for NMF.\n",
    "print(\"Extracting tf-idf features for NMF...\")\n",
    "tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,\n",
    "                                   max_features=n_features)\n",
    "t0 = time()\n",
    "tfidf = tfidf_vectorizer.fit_transform(documents)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use tf (raw term count) features for LDA.\n",
    "print(\"Extracting tf features for LDA...\")\n",
    "tf_vectorizer = CountVectorizer(max_df=0.95, min_df=2,\n",
    "                                max_features=n_features)\n",
    "t0 = time()\n",
    "tf = tf_vectorizer.fit_transform(documents)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the NMF model\n",
    "print(\"Fitting the NMF model with tf-idf features, \"\n",
    "      \"n_samples=%d and n_features=%d...\"\n",
    "      % (n_samples, n_features))\n",
    "t0 = time()\n",
    "nmf = NMF(n_components=n_topics, random_state=1,\n",
    "          alpha=.1, l1_ratio=.5).fit(tfidf)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\nTopics in NMF model:\")\n",
    "tfidf_feature_names = tfidf_vectorizer.get_feature_names()\n",
    "print_top_words(nmf, tfidf_feature_names, n_top_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Fitting LDA models with tf features, \"\n",
    "      \"n_samples=%d and n_features=%d...\"\n",
    "      % (n_samples, n_features))\n",
    "lda = LatentDirichletAllocation(n_topics=n_topics, max_iter=5,\n",
    "                                learning_method='online',\n",
    "                                learning_offset=50.,\n",
    "                                random_state=0)\n",
    "t0 = time()\n",
    "lda.fit(tf)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\nTopics in LDA model:\")\n",
    "tf_feature_names = tf_vectorizer.get_feature_names()\n",
    "print_top_words(lda, tf_feature_names, n_top_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Retweets</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "retweets.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped = retweets.groupby(['parentPostAuthor','parentPost']).size().reset_index()\n",
    "grouped.rename(columns={0:'counts'},inplace=True)\n",
    "grouped.sort_values('counts', ascending=False, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_retweeted = grouped.iloc[:5,:]\n",
    "most_retweeted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_retweeted = tweets.loc[tweets.snspostid.isin(grouped.iloc[:100,:].parentPost.astype(np.str).values)]\n",
    "most_retweeted.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "least_retweeted = tweets.loc[tweets.snspostid.isin(grouped.iloc[-100:,:].parentPost.astype(np.str).values)]\n",
    "least_retweeted.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for id,row in tweets.loc[tweets.snspostid.isin(least_retweeted.parentPost.astype(np.str).values)].iterrows():\n",
    "    print(row.text)\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = ' '.join(most_retweeted.text.values.tolist())\n",
    "for word in all_stopwords:\n",
    "    text = text.lower().replace(' {} '.format(word),'')\n",
    "text = text.replace('?','').replace('.','').replace(',','').replace(':','').replace(';','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "makeImage(getFrequencyDictForText(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install nltk"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}