explore_hashtags.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "import numpy as np\n",
    "from glob import glob\n",
    "import pandas as pd\n",
    "from collections import defaultdict, Counter\n",
    "import metis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfs = [pd.read_pickle(p) for p in glob('data/retweets_16july/*.pkl')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.concat(dfs, axis=0)\n",
    "if False:\n",
    "    df.to_pickle('data/july.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df['hashtag'].value_counts()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tag = 'MTVHottest'\n",
    "hdf = df[df['hashtag'] == tag]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_graph_from_df(df):\n",
    "    g = nx.Graph()\n",
    "    for _, r in df.iterrows():\n",
    "        g.add_edge(r['retweeter'], r['retweetee'])\n",
    "    return g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "subg = build_graph_from_df(hdf)\n",
    "nx.write_gpickle(subg, 'data/{}.gpkl'.format(tag.lower()))\n",
    "hdf.to_pickle('data/{}.pkl'.format(tag.lower()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "describe_graph(subg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from rwc import controversy_score\n",
    "print('rwc score {}'.format(controversy_score(subg)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_pickle('data/retweets_test.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "retweets_by_hashtags = defaultdict(list)\n",
    "for i, r in df.iterrows():    \n",
    "    for h in r['hashtags']:\n",
    "        retweets_by_hashtags[h].append(r.to_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tag2freq = Counter({h: len(retweets_by_hashtags[h])\n",
    "                    for h in retweets_by_hashtags})\n",
    "# print(tag2freq.most_common(10))\n",
    "top_tags = [h for h, _ in tag2freq.most_common(10)]\n",
    "print('Top 10 frequent tags: {}'.format(top_tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_graph_from_retweets(rts):\n",
    "    g = nx.Graph()  # QUESTION: directed or undirected?\n",
    "    g.add_edges_from((r['retweeter'], r['retweetee']) for r in rts)\n",
    "    return g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def describe_graph(g):\n",
    "    components = list(nx.connected_components(g))\n",
    "    top_cc_sizes = list(sorted(map(len, components), reverse=True))[:3]\n",
    "    top_cc_ratios = np.array(top_cc_sizes) / g.number_of_nodes()\n",
    "    print(\"\"\"\n",
    "    #nodes: {},\n",
    "    #edges: {},\n",
    "    size of top-3 component: {}\n",
    "    ratio of top-3 components: {}\n",
    "    \"\"\".format(\n",
    "            g.number_of_nodes(),\n",
    "            g.number_of_edges(),\n",
    "            top_cc_sizes,\n",
    "            top_cc_ratios\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "graph_by_hashtag = {\n",
    "    h: build_graph_from_retweets(retweets_by_hashtags[h])\n",
    "    for h in top_tags}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# some general statistics\n",
    "for h, g in list(graph_by_hashtag.items())[:]:\n",
    "    print(\"Hashtag: {}\".format(h))\n",
    "    describe_graph(g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def largest_connected_subgraph(g):\n",
    "    ccs = nx.connected_components(g)\n",
    "    nodes = max(ccs, key=len)\n",
    "    return g.subgraph(nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for h, g in graph_by_hashtag.items():\n",
    "    subg = largest_connected_subgraph(g)\n",
    "    cuts, parts = metis.part_graph(subg, 2)\n",
    "    print(\"\"\"hashtag: {}\n",
    "    cut ratio: {}\n",
    "    \"\"\".format(h,\n",
    "               cuts / subg.number_of_edges()))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}