twitter_api_tweet_analysis_part2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Twitter data part 2\n",
    "\n",
    "The API basics can be found in [Part 1](./twitter_api_tweet_analysis.ipynb)\n",
    "\n",
    "<div class=note><b>Copyright and Licensing:</b>\n",
    "\n",
    "\n",
    "You are free to use or adapt this notebook for any purpose you'd like. However, please respect the [Simplified BSD License](https://github.com/ptwobrussell/Mining-the-Social-Web-2nd-Edition/blob/master/LICENSE.txt) that governs its use.</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load ../_data/standard_import.txt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import pickle\n",
    "import twitter\n",
    "\n",
    "plt.style.use('seaborn-white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_file = '../_credentials/twitter_credentials.pkl'\n",
    "Twitter=pickle.load(open(pickle_file,'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Authorizing an application to access Twitter account data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth = twitter.oauth.OAuth(Twitter['Access Token'],\n",
    "                           Twitter['Access Token Secret'],\n",
    "                           Twitter['Consumer Key'],\n",
    "                           Twitter['Consumer Secret'])\n",
    "\n",
    "twitter_api = twitter.Twitter(auth=auth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Yahoo! Where On World ID dictionary\n",
    "\n",
    "The Yahoo! Where On Earth ID for the entire world is 1.  \n",
    "[Find your WOE ID](http://woeid.rosselliot.co.nz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_woeid(locations):\n",
    "    import requests\n",
    "    from bs4 import BeautifulSoup\n",
    "    woeids = {}\n",
    "    if type(locations) != type([]): \n",
    "        locations = [locations]\n",
    "    for loc in locations:\n",
    "        try:\n",
    "            response = requests.get('http://woeid.rosselliot.co.nz/lookup/' + loc)\n",
    "            results_page = BeautifulSoup(response.content, 'lxml')\n",
    "            woeids[loc] = int(results_page.find_all('td', {\"class\": \"woeid\"})[0].text)\n",
    "        except:\n",
    "            continue\n",
    "    return woeids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOCATIONS = ['World', 'United States', 'Netherlands', 'Europe', 'Amsterdam', 'Abcoude']\n",
    "WOEID = get_woeid(LOCATIONS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for loc in LOCATIONS:\n",
    "    print(loc, WOEID[loc])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trends by tweet volume"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def woe_trends_(location):\n",
    "    \"\"\"Returns localised trends\"\"\"\n",
    "    WOEID = get_woeid(location)\n",
    "    local_trends = twitter_api.trends.place(_id=WOEID[location])[0]['trends']\n",
    "    return (pd.DataFrame([(trend['name'],trend['tweet_volume']) \n",
    "                         for trend in local_trends], \n",
    "                         columns=['trend', 'volume'])\n",
    "                         .sort_values('volume', ascending=False)\n",
    "                         .set_index('trend'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "woe_trends_('Amsterdam').head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Locality of trends"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trending_location(location):\n",
    "    \"\"\"Returns localised trends\"\"\"\n",
    "    WOE_ID = get_woeid(location)\n",
    "    local_trends = twitter_api.trends.place(_id=WOE_ID[location])[0]['trends']\n",
    "    return set([trend['name'] for trend in local_trends])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trending_shared(locations):\n",
    "    \"\"\"Returns trends shared between WOE_ID_1 & WOE_ID_2\"\"\"\n",
    "    WOE_ID = get_woeid(locations)\n",
    "    local_trends_1 = twitter_api.trends.place(_id=WOE_ID[locations[0]])[0]['trends']\n",
    "    local_trends_2 = twitter_api.trends.place(_id=WOE_ID[locations[1]])[0]['trends']\n",
    "    set_1 = set([trend['name'] for trend in local_trends_1])\n",
    "    set_2 = set([trend['name'] for trend in local_trends_2])\n",
    "    return list(set_1 & set_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trending_excl(locations):\n",
    "    \"\"\"Returns trends only in WOE_ID_1\"\"\"\n",
    "    WOE_ID = get_woeid(locations)\n",
    "    local_trends_1 = twitter_api.trends.place(_id=WOE_ID[locations[0]])[0]['trends']\n",
    "    local_trends_2 = twitter_api.trends.place(_id=WOE_ID[locations[1]])[0]['trends']\n",
    "    set_1 = set([trend['name'] for trend in local_trends_1])\n",
    "    set_2 = set([trend['name'] for trend in local_trends_2])\n",
    "    return list((set_1 ^ set_2) & set_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trends_set = {}\n",
    "trends_set['world'] = trending_location('World')\n",
    "trends_set['nl'] = trending_location('Netherlands')\n",
    "trends_set['amsterdam'] = trending_location('Amsterdam')\n",
    "trends_set['nl&ams'] = trending_shared(['Netherlands', 'Amsterdam'])\n",
    "trends_set['ams^nl'] = trending_excl(['Amsterdam', 'Netherlands'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trends_set['ams^nl']\n",
    "trends_set['ams^nl'][0]\n",
    "trends_set['nl&ams'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trending_shared(['Netherlands', 'Germany'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collecting search results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set the variable `q` to a trending topic, \n",
    "or anything else for that matter. The example query below\n",
    "was a trending topic when this content was being developed\n",
    "and is used throughout the remainder of this chapter\n",
    "\n",
    "[api docs](https://dev.twitter.com/docs/api/1.1/get/search/tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recursive_nodes(x, **kwargs):\n",
    "    \"\"\"\"\"\"\n",
    "    key = kwargs.get('key', None)\n",
    "    y = kwargs.get('y', None)\n",
    "    \n",
    "    # base case\n",
    "    if y is None: y = []\n",
    "    if x == []: return y\n",
    "    \n",
    "    # recursive call\n",
    "    if key is None: \n",
    "        y.append(x[0])\n",
    "#         recursive_nodes(x[1:], None, y)\n",
    "    else: \n",
    "        y.append(x[0][key])\n",
    "    recursive_nodes(x[1:], key=key, y=y)\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert recursive_nodes([1,2,3,4]) == [1, 2, 3, 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trending_topic(topic, number=100):\n",
    "    \"\"\"Returns status\"\"\"\n",
    "    return twitter_api.search.tweets(q=topic, count=number)['statuses']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trending_topic(topic, number=100):\n",
    "    \"\"\"Returns status\"\"\"\n",
    "    return twitter_api.search.tweets(q=topic, count=number)['statuses']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trending_text(topic, number=100, head=10):\n",
    "    \"\"\"Returns status text\"\"\"\n",
    "    tweet_ = [(s['user']['screen_name'], recursive_nodes(s['entities']['user_mentions'], key='screen_name'),\n",
    "               recursive_nodes(s['entities']['hashtags'], key='text'), s['text'], s['retweet_count']) \n",
    "              for s in trending_topic(topic)]\n",
    "    return pd.DataFrame(tweet_, columns=['name', 'mentions', 'hashtags', 'text', 'retweets']).sort_values('retweets', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic = trends_set['ams^nl'][0]\n",
    "statuses = trending_topic(topic)\n",
    "statuses[0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "statuses[0]#['geo']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = trending_text(topic, number=100, head=10)\n",
    "df.head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.mentions[:10]\n",
    "df.hashtags[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extracting text, screen names, and hashtags from tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# status_texts = [status['text'] for status in statuses]\n",
    "\n",
    "# screen_names = [user_mention['screen_name'] for status in statuses\n",
    "#                                             for user_mention in status['entities']['user_mentions']]\n",
    "\n",
    "# hashtags = [hashtag['text'].lower() for status in statuses\n",
    "#                             for hashtag in status['entities']['hashtags']]\n",
    "stopwords = ['de', 'het', 'een', 'is', 'de', 'die', 'dat', 'dit', 'van', 'en', 'rt', 'in', 'er', 'op', 'als', 'aan', 'als', 'bij',\n",
    "             'met', 'niet', 'voor', 'gaat', 'ze', 'je', 'ik', 'wij', 'rt', 'staan', 'kan', 'dan', 'af', 'zoals', 'laat', 'naar',\n",
    "             'meer', 'werd', 'geen', 'na', 'heeft', 'komt', 'wel', 'nog', 'over', '-']\n",
    "\n",
    "# Compute a collection of all words from all tweets\n",
    "words = [w.lower() for t in df['text']\n",
    "           for w in t.split() if w.lower() not in stopwords]\n",
    "Counter(words).most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "status_texts[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Explore the first 5 items for each...\n",
    "print('status text: ', json.dumps(status_texts[0:5], indent=1))\n",
    "print('screen names: ', json.dumps(screen_names[0:5], indent=1)) \n",
    "print('hashtags: ', json.dumps(hashtags[0:5], indent=1))\n",
    "print('words: ', json.dumps(words[0:5], indent=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic frequency distribution from the words in tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "for item in words: #[words, screen_names, hashtags]:\n",
    "    c = Counter(item)\n",
    "    print('-'*80)\n",
    "    print(c.most_common()[:10]) # top 10\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(Counter(words).most_common(30), columns=['word', 'count']).set_index('word').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(Counter(screen_names).most_common(30), columns=['mentions', 'count']).set_index('mentions').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(Counter(hashtags).most_common(30), columns=['hashtags', 'count']).set_index('hashtags').head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Most popular retweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "retweets = [\n",
    "            # Store out a tuple of these three values ...\n",
    "            (status['retweet_count'], \n",
    "             status['retweeted_status']['user']['screen_name'],\n",
    "             status['text'].replace(\"\\n\",\"\\\\\")) \n",
    "            \n",
    "            # ... for each status ...\n",
    "            for status in statuses \n",
    "            \n",
    "            # ... so long as the status meets this condition.\n",
    "                if 'retweeted_status' in status\n",
    "           ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_retweets = pd.DataFrame(retweets, columns=['retweets', 'screen_name', 'text']).sort_values('retweets', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_retweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_retweets.text[16]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}