create-tweet-hashtag-dashboard.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''Download and analyze Twitter data stored in MongoDB\n",
    "'''\n",
    "\n",
    "# standard library\n",
    "import configparser\n",
    "from datetime import date, timedelta\n",
    "import os\n",
    "\n",
    "# non-standard library\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pymongo import MongoClient\n",
    "import pytz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mongo Functions\n",
    "\n",
    "### Two ways:\n",
    "1. ```try...finally```\n",
    "2. Context Manager"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Without using a context manager\n",
    "def download_tweets_by_hashtag_nonpythonic(hashtag):\n",
    "    '''Connect to MongoDB, download tweets with param hashtags\n",
    "    \n",
    "    Args:\n",
    "        * hashtag - text hashtag\n",
    "        \n",
    "    Returns\n",
    "        * list of Tweets containing hashtag\n",
    "    '''\n",
    "    \n",
    "    tweets = []\n",
    "    try:\n",
    "        client = MongoClient(mlab_uri)\n",
    "        db = client.get_default_database()\n",
    "        coll = db[collection]\n",
    "        tweets = coll.find({\"entities.hashtags.text\":f\"{hashtag}\"})\n",
    "    finally:\n",
    "        client.close()\n",
    "        return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MongoCollection(object):\n",
    "    '''Connect to mongodb and return collection within context manager\n",
    "    \n",
    "    http://book.pythontips.com/en/latest/context_managers.html\n",
    "    '''\n",
    "    \n",
    "    def __init__(self, uri, collection):\n",
    "        self.client = MongoClient(uri)\n",
    "        self.db = self.client.get_default_database()\n",
    "        self.collection = self.db[collection]\n",
    "        \n",
    "    def __enter__(self):\n",
    "        return self.collection\n",
    "    \n",
    "    def __exit__(self, exc_type, exc_value, exc_traceback):\n",
    "        self.client.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_tweets_by_hashtag(hashtag):\n",
    "    '''Connect to MongoDB, download tweets with param hashtags\n",
    "    \n",
    "    Args:\n",
    "        * hashtag - text hashtag\n",
    "        \n",
    "    Returns\n",
    "        * list of Tweets containing hashtag\n",
    "    '''\n",
    "    with MongoCollection(mlab_uri, collection) as coll:\n",
    "        tweets = coll.find({'entities.hashtags.text':f'{hashtag}'})\n",
    "    \n",
    "    return list(tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Get data from Mongo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get mongodb params (using configparser)\n",
    "config = configparser.ConfigParser()\n",
    "config_file = 'settings.cfg'\n",
    "config.read(config_file)\n",
    "mlab_uri = config.get('MongoDB', 'mlab_uri')\n",
    "collection = config.get('MongoDB', 'mlab_collection')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# download tweets\n",
    "tweets = download_tweets_by_hashtag('codeeveryday')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spent around 30 minutes working on my side project this morning. Not sure if I'll start a commit streak, but will #codeeveryday #programming\n",
      "Finished an analysis of historical Premier League results. Blog post coming shortly #codeeveryday\n",
      "Got into the office early to work on the side project. Starbucks after work to finish off today's task #codeeveryday\n",
      "Early to bed, early to rise. Refactored my analysis to use a #postgres backend, set up as a script. And done! Next project! #codeeveryday\n",
      "Little late in the day, but was able to fit some time in for the side project. #codeeveryday\n",
      "Early morning bash scripting. #!/bin/bash/codeeveryday #codeeveryday #programming #unix #linux #macos #chmod #chmod755\n",
      "Finished up the project I started yesterday. Productive weekend. #codeeveryday\n",
      "Finished a blog draft. Does Markdown count? I say yes. Any kind of progress on a side project, blogs included. #codeeveryday\n",
      "Another day working on the blog. #codeeveryday\n",
      "Playing around on #Bluemix, trying to figure out the best workflow for my new project. #codeeveryday\n",
      "Refactored some code to make it easier to read / edit going forward. Save developer hours vs computer seconds #codeeveryday\n",
      "Refactored work from last night into a function. Really want to finish, but should spend some time thinking about the problem. #codeeveryday\n",
      "Building upon some previous work I did. Trying to create some interesting visualizations. #codeeveryday\n",
      "Playing around in terminal trying to get #bsonnumpy installed. No real progress, will use an ORM and revisit later #codeeveryday\n",
      "Set up pymongo connection code inside context manager to make querying more #pythonic. #codeeveryday\n",
      "Took a very late lunch to put finishing touches on the latest blog post. #codeeveryday\n",
      "As part of today's #codeeveryday exercise, I wrote some #Python to analyze my #codeeveryday tweets. So meta. https://t.co/rti8Z9O34r\n",
      "Drafted up a question for #stackoverflow and @ChicagoPython Meetup. #codeeveryday It counts.\n",
      "Took a break from side projects yesterday. Back at it. #codeeveryday\n",
      "Spent the morning relearning #Spark. DataFrames &gt;&gt; RDD, but I do miss thinking functionally. #lambda #functionalprogramming #codeeveryday\n",
      "Watching hockey and working on the demo for my presentation. #HIVE #Spark #Hadoop #hdfs #codeeveryday\n",
      "Working on my slide desk and demo. #codeeveryday\n",
      "Worked on my presentation. Too much hockey and not enough coding. Will need to wake up early to catchup. #codeeveryday\n",
      ".@ChicagoPython Project Night at @braintree. Breakthrough on my viz project. Trying a new approach figured out via collab. #codeeveryday\n",
      "Working on my mapping demo. Folium 0.3 docs need to be updated, I'll write up a gist for what I figured out next week #codeeveryday\n",
      "React workshop. Getting back into #frontend #javascript. #codeeveryday\n",
      "Working on slides #codeeveryday\n",
      "Finally done my slides. Just need to write instructions on how to replicate my demo. Docker...? Maybe next time #codeeveryday\n",
      "Gave my talk today. Back to projects tomorrow. #codeeveryday\n",
      "Working on an HTML visualization. Relearning CSS. #wannabedesigner #codeeveryday\n",
      "Playing around with context managers. Drafting up an outline for the blog. #codeeveryday\n",
      "Installed DC/OS and got it running. #codeeveryday https://t.co/9IMk4JVtZT\n",
      "Playing around with DCOS. Got an interesting project in mind, but I need to learn how to connect the dots first. #codeeveryday\n",
      "Finished a blog post. Sent to my editors. #codeeveryday\n",
      "Learned a bit of TDD web dev. Shoutout to @hjwp. Helped a buddy modularize his program. #encapsulation #codeeveryday\n",
      "Going deeper on the blog entry from earlier in the week. Researching and reading. #codeeveryday\n"
     ]
    }
   ],
   "source": [
    "for tweet in tweets:\n",
    "    print(tweet['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_id': ObjectId('58d92c3e598402000101e989'),\n",
       " 'contributors': None,\n",
       " 'coordinates': None,\n",
       " 'created_at': 'Mon Mar 27 13:07:10 +0000 2017',\n",
       " 'entities': {'hashtags': [{'indices': [114, 127], 'text': 'codeeveryday'},\n",
       "   {'indices': [128, 140], 'text': 'programming'}],\n",
       "  'symbols': [],\n",
       "  'urls': [],\n",
       "  'user_mentions': []},\n",
       " 'favorite_count': 1,\n",
       " 'favorited': False,\n",
       " 'geo': None,\n",
       " 'id': 846347868734722049,\n",
       " 'id_str': '846347868734722049',\n",
       " 'in_reply_to_screen_name': None,\n",
       " 'in_reply_to_status_id': None,\n",
       " 'in_reply_to_status_id_str': None,\n",
       " 'in_reply_to_user_id': None,\n",
       " 'in_reply_to_user_id_str': None,\n",
       " 'is_quote_status': False,\n",
       " 'lang': 'en',\n",
       " 'place': None,\n",
       " 'retweet_count': 0,\n",
       " 'retweeted': False,\n",
       " 'source': '<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>',\n",
       " 'text': \"Spent around 30 minutes working on my side project this morning. Not sure if I'll start a commit streak, but will #codeeveryday #programming\",\n",
       " 'truncated': False,\n",
       " 'user': {'contributors_enabled': False,\n",
       "  'created_at': 'Wed Dec 26 18:37:29 +0000 2012',\n",
       "  'default_profile': False,\n",
       "  'default_profile_image': False,\n",
       "  'description': 'IBMer. #python, #healthIT, #healthdata, #datascience, #cycling, #running, #soccer, #startrek. Current: @TruvenHealth @NorthwesternMMI. Past: @WaterlooMath.',\n",
       "  'entities': {'description': {'urls': []},\n",
       "   'url': {'urls': [{'display_url': 'alysivji.github.io',\n",
       "      'expanded_url': 'https://alysivji.github.io',\n",
       "      'indices': [0, 23],\n",
       "      'url': 'https://t.co/thfgWBPb7l'}]}},\n",
       "  'favourites_count': 760,\n",
       "  'follow_request_sent': False,\n",
       "  'followers_count': 521,\n",
       "  'following': False,\n",
       "  'friends_count': 1588,\n",
       "  'geo_enabled': True,\n",
       "  'has_extended_profile': False,\n",
       "  'id': 1037776201,\n",
       "  'id_str': '1037776201',\n",
       "  'is_translation_enabled': False,\n",
       "  'is_translator': False,\n",
       "  'lang': 'en',\n",
       "  'listed_count': 92,\n",
       "  'location': 'Chicago',\n",
       "  'name': 'Aly Sivji',\n",
       "  'notifications': False,\n",
       "  'profile_background_color': 'C0DEED',\n",
       "  'profile_background_image_url': 'http://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_image_url_https': 'https://abs.twimg.com/images/themes/theme1/bg.png',\n",
       "  'profile_background_tile': False,\n",
       "  'profile_banner_url': 'https://pbs.twimg.com/profile_banners/1037776201/1401051600',\n",
       "  'profile_image_url': 'http://pbs.twimg.com/profile_images/564558858224664576/s6s8dm3G_normal.jpeg',\n",
       "  'profile_image_url_https': 'https://pbs.twimg.com/profile_images/564558858224664576/s6s8dm3G_normal.jpeg',\n",
       "  'profile_link_color': '0084B4',\n",
       "  'profile_sidebar_border_color': 'C0DEED',\n",
       "  'profile_sidebar_fill_color': 'DDEEF6',\n",
       "  'profile_text_color': '333333',\n",
       "  'profile_use_background_image': True,\n",
       "  'protected': False,\n",
       "  'screen_name': 'CaiusSivjus',\n",
       "  'statuses_count': 1094,\n",
       "  'time_zone': None,\n",
       "  'translator_type': 'none',\n",
       "  'url': 'https://t.co/thfgWBPb7l',\n",
       "  'utc_offset': None,\n",
       "  'verified': False}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean data and throw into Pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "central = pytz.timezone('US/Central')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cleaned_tweets = []\n",
    "\n",
    "# go through tweets, extract (and transform) fields into useable format\n",
    "for tweet in tweets:\n",
    "    item = {}\n",
    "    item['text'] = tweet['text']\n",
    "    \n",
    "    # convert from UTC to CST and drop timezone as it is not relevant anymore\n",
    "    tweet_time = pd.to_datetime(tweet['created_at'])\n",
    "    utc_dt = pytz.utc.localize(tweet_time)\n",
    "    central_dt = utc_dt.astimezone(central).to_pydatetime()\n",
    "    tweet_time = central_dt.replace(tzinfo=None)\n",
    "    tweet_date = tweet_time.replace(hour=0, minute=0, second=0)\n",
    "    \n",
    "    item['created_at'] = tweet_time\n",
    "    item['date'] = central_dt.date()\n",
    "    item['time'] = central_dt.time()\n",
    "    item['timedelta'] = tweet_time - tweet_date\n",
    "    item['count'] = 1 ## needed?\n",
    "    \n",
    "    cleaned_tweets.append(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tweets_df = pd.DataFrame(cleaned_tweets).sort_values('date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>created_at</th>\n",
       "      <th>date</th>\n",
       "      <th>text</th>\n",
       "      <th>time</th>\n",
       "      <th>timedelta</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2017-03-27 08:07:10</td>\n",
       "      <td>2017-03-27</td>\n",
       "      <td>Spent around 30 minutes working on my side pro...</td>\n",
       "      <td>08:07:10</td>\n",
       "      <td>08:07:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2017-03-28 07:51:32</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>Finished an analysis of historical Premier Lea...</td>\n",
       "      <td>07:51:32</td>\n",
       "      <td>07:51:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2017-03-29 08:47:37</td>\n",
       "      <td>2017-03-29</td>\n",
       "      <td>Got into the office early to work on the side ...</td>\n",
       "      <td>08:47:37</td>\n",
       "      <td>08:47:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2017-03-30 06:11:29</td>\n",
       "      <td>2017-03-30</td>\n",
       "      <td>Early to bed, early to rise. Refactored my ana...</td>\n",
       "      <td>06:11:29</td>\n",
       "      <td>06:11:29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   count          created_at        date  \\\n",
       "0      1 2017-03-27 08:07:10  2017-03-27   \n",
       "1      1 2017-03-28 07:51:32  2017-03-28   \n",
       "2      1 2017-03-29 08:47:37  2017-03-29   \n",
       "3      1 2017-03-30 06:11:29  2017-03-30   \n",
       "\n",
       "                                                text      time  timedelta  \n",
       "0  Spent around 30 minutes working on my side pro...  08:07:10   08:07:10  \n",
       "1  Finished an analysis of historical Premier Lea...  07:51:32   07:51:32  \n",
       "2  Got into the office early to work on the side ...  08:47:37   08:47:37  \n",
       "3  Early to bed, early to rise. Refactored my ana...  06:11:29   06:11:29  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.head(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def on_streak(df, date_to_check):\n",
    "    '''Check to see if we tweeted hashtag on that day\n",
    "    \n",
    "    Args:\n",
    "        * df with dateindex\n",
    "        * day to check\n",
    "        \n",
    "    Returns:\n",
    "        Boolean\n",
    "    '''\n",
    "    \n",
    "    if date_to_check in df.index:\n",
    "        return df.loc[date_to_check].indicator == 1\n",
    "    \n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get counts for each day, set date as index\n",
    "counts_df = pd.DataFrame(tweets_df[['created_at', 'count']])\n",
    "counts_df = counts_df.set_index('created_at')\n",
    "counts_df.index.to_datetime\n",
    "\n",
    "# sum up each day's count and fill days with no activity with 0\n",
    "counts_df = counts_df.resample('D').sum().fillna(0)\n",
    "counts_df = pd.DataFrame(np.where(counts_df['count'] > 0, 1, 0),\n",
    "                         index=counts_df.index,\n",
    "                         columns=['indicator'])\n",
    "\n",
    "# find streaks\n",
    "counts_df['block'] = ((counts_df != counts_df.shift(1))\n",
    "                          .astype(int)\n",
    "                          .cumsum())\n",
    "\n",
    "# get length of streaks\n",
    "streak_lengths = (counts_df\n",
    "                      .reset_index()\n",
    "                      .groupby(['indicator', 'block'])['created_at']\n",
    "                      .apply(np.size))\n",
    "\n",
    "# pull the level where indicator=1\n",
    "longest_streak = streak_lengths[1].max()\n",
    "num_days_with_tweet = streak_lengths[1].sum()\n",
    "total_tweets = len(tweets_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# current streak\n",
    "last_day_of_streak = None\n",
    "tweet_to_continue_streak = False\n",
    "\n",
    "# check if we are currently in the middle of a streak\n",
    "today = date.today()\n",
    "if on_streak(counts_df, today):\n",
    "    last_day_of_streak = date.today()\n",
    "\n",
    "# since tweets are downloaded every 12 hours, we might not have today's info\n",
    "# check if we were on a streak yesterday\n",
    "if last_day_of_streak == None:\n",
    "    yesterday = (date.today() - timedelta(days=1))\n",
    "\n",
    "    if on_streak(counts_df, yesterday):\n",
    "        last_day_of_streak = yesterday\n",
    "        tweet_to_continue_streak = True  # remind user to tweet to continue streak!\n",
    "    \n",
    "# if we are currently streaking, count \n",
    "curr_streak = 0\n",
    "if last_day_of_streak:\n",
    "    day_to_check = last_day_of_streak\n",
    "    while on_streak(counts_df, day_to_check):\n",
    "        curr_streak += 1\n",
    "        day_to_check -= timedelta(days=1) # go back a day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweet Time Stats\n",
      "----------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "count                        36\n",
       "mean     0 days 16:57:48.583333\n",
       "std      0 days 06:08:23.534578\n",
       "min             0 days 06:11:29\n",
       "25%      0 days 10:14:08.500000\n",
       "50%      0 days 20:37:23.500000\n",
       "75%             0 days 21:53:44\n",
       "max             0 days 23:47:08\n",
       "Name: timedelta, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Tweet Time Stats')\n",
    "print('----------------')\n",
    "tweets_df['timedelta'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweet Date Results\n",
      "------------------\n",
      "Total tweets referencing hashtag: 36\n",
      "Number of days hashtag was used: 36\n",
      "Most consecutive days using hastag: 18\n",
      "Current Streak: 18\n",
      "Need to tweet to continue streak? True\n"
     ]
    }
   ],
   "source": [
    "print('Tweet Date Results')\n",
    "print('------------------')\n",
    "print(f'Total tweets referencing hashtag: {total_tweets}')\n",
    "print(f'Number of days hashtag was used: {num_days_with_tweet}')\n",
    "print(f'Most consecutive days using hastag: {longest_streak}')\n",
    "print(f'Current Streak: {curr_streak}')\n",
    "print(f'Need to tweet to continue streak? {tweet_to_continue_streak}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## TODO:\n",
    "# use itertools to count longest streak. FUN EXERCISE\n",
    "# http://stackoverflow.com/questions/24342047/count-consecutive-occurences-of-values-varying-in-length-in-a-numpy-array\n",
    "\n",
    "# do a for from today/yesterday to get current streak"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}