The Github Archive Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'C:\\\\ARGO\\\\talks\\\\split-apply-combine'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8.3\n",
      "actor:\n",
      "  avatar_url: https://avatars.githubusercontent.com/u/9152315?\n",
      "  gravatar_id: ''\n",
      "  id: 9152315\n",
      "  login: davidjhulse\n",
      "  url: https://api.github.com/users/davidjhulse\n",
      "created_at: '2015-01-01T00:00:00Z'\n",
      "id: '2489368070'\n",
      "payload:\n",
      "  before: 86ffa724b4d70fce46e760f8cc080f5ec3d7d85f\n",
      "  commits:\n",
      "  - author:\n",
      "      email: david.hulse@live.com\n",
      "      name: davidjhulse\n",
      "    distinct: true\n",
      "    message: 'Altered BingBot.jar\n",
      "\n",
      "\n",
      "      Fixed issue with multiple account support'\n",
      "    sha: a9b22a6d80c1e0bb49c1cf75a3c075b642c28f81\n",
      "    url: https://api.github.com/repos/davidjhulse/davesbingrewardsbot/commits/a9b22a6d80c1e0bb49c1cf75a3c075b642c28f81\n",
      "  distinct_size: 1\n",
      "  head: a9b22a6d80c1e0bb49c1cf75a3c075b642c28f81\n",
      "  push_id: 536740396\n",
      "  ref: refs/heads/master\n",
      "  size: 1\n",
      "public: true\n",
      "repo:\n",
      "  id: 28635890\n",
      "  name: davidjhulse/davesbingrewardsbot\n",
      "  url: https://api.github.com/repos/davidjhulse/davesbingrewardsbot\n",
      "type: PushEvent\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import gzip\n",
    "import json\n",
    "import yaml\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import blaze as bz\n",
    "print bz.__version__\n",
    "\n",
    "directory = r'data\\github_archive'\n",
    "filename = '2015-01-01-0.json.gz'\n",
    "path = os.path.join(directory, filename)\n",
    "with gzip.open(path) as f:\n",
    "        print yaml.dump(yaml.load(f.readline()), default_flow_style=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CommitCommentEvent: 56\n",
      "CreateEvent: 815\n",
      "DeleteEvent: 141\n",
      "ForkEvent: 213\n",
      "GollumEvent: 90\n",
      "IssueCommentEvent: 650\n",
      "IssuesEvent: 373\n",
      "MemberEvent: 16\n",
      "PublicEvent: 2\n",
      "PullRequestEvent: 315\n",
      "PullRequestReviewCommentEvent: 85\n",
      "PushEvent: 4280\n",
      "ReleaseEvent: 24\n",
      "WatchEvent: 642\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "import ujson\n",
    "with gzip.open(path) as f:\n",
    "    counts = Counter(str(ujson.loads(line)['type']) for line in f)\n",
    "print yaml.dump(dict(counts), default_flow_style=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7702\n",
      "815\n",
      "4280\n",
      "315\n"
     ]
    }
   ],
   "source": [
    "with gzip.open(path) as f:\n",
    "    events = [ujson.loads(line) for line in f]\n",
    "creates = [event for event in events if event['type']=='CreateEvent']\n",
    "pushes = [event for event in events if event['type']=='PushEvent']\n",
    "pull_requests = [event for event in events if event['type']=='PullRequestEvent']\n",
    "print len(events)\n",
    "print len(creates)\n",
    "print len(pushes)\n",
    "print len(pull_requests)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'tridge', 5), (u'interactivellama', 4), (u'we555', 4), (u'fishulla', 4), (u'vegitron', 4), (u'tianon', 4), (u'mike-spainhower', 4), (u'rohanbk', 3), (u'mdamt', 3), (u'cspray', 3)]\n",
      "[(u'selfhub/selfhub', 5), (u'tridge/cuav', 5), (u'docker-library/docs', 4), (u'eclipse/birt', 4), (u'Merchello/Merchello', 4), (u'TeamGabriel/gabriel', 4), (u'twbs/bootstrap', 4), (u'uw-it-aca/jira-hook', 4), (u'fishulla/Torque3D', 4), (u'we555/hello-world', 4)]\n"
     ]
    }
   ],
   "source": [
    "print Counter(pr['actor']['login'] for pr in pull_requests).most_common()[:10]\n",
    "print Counter(pr['repo']['name'] for pr in pull_requests).most_common()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'KenanSulayman', 79), (u'opencm', 66), (u'mirror-updates', 42), (u'cm-gerrit', 35), (u'westurner', 30), (u'qdm', 29), (u'NikhilSingh19', 27), (u'greatfire', 24), (u'ejoseca', 24), (u'tianon', 21)]\n"
     ]
    }
   ],
   "source": [
    "actors = Counter(event['actor']['login'] for event in events)\n",
    "print actors.most_common()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'KenanSulayman/heartbeat', 79), (u'sakai-mirror/melete', 30), (u'qdm/qdm.github.io', 29), (u'ejoseca/great-validator', 24), (u'square/okhttp', 23), (u'westurner/sphinxcontrib-srclinks', 23), (u'captainkirkby/Gears', 21), (u'apache/spark', 21), (u'docker-library/docs', 21), (u'NikhilSingh19/NikhilSingh19.github.io', 20)]\n"
     ]
    }
   ],
   "source": [
    "repos = Counter(event['repo']['name'] for event in events)\n",
    "print repos.most_common()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'master_branch': u'master', u'ref_type': u'branch', u'ref': u'master', u'description': u'Making an HTML5 game from scratch.', u'pusher_type': u'user'}\n",
      "{u'master_branch': u'master', u'ref_type': u'branch', u'ref': u'master2', u'description': u'Deploy and Manage Docker Container', u'pusher_type': u'user'}\n",
      "{u'master_branch': u'master', u'ref_type': u'repository', u'ref': None, u'description': u'An ansible role for setting up Wordpress Nginx website', u'pusher_type': u'user'}\n",
      "{u'master_branch': u'master', u'ref_type': u'tag', u'ref': u'0.0.184', u'description': None, u'pusher_type': u'user'}\n",
      "{u'master_branch': u'master', u'ref_type': u'tag', u'ref': u'0.1.32', u'description': None, u'pusher_type': u'user'}\n"
     ]
    }
   ],
   "source": [
    "for event in creates[:5]:\n",
    "    print event['payload']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"payload\": {\n",
      "        \"size\": 1, \n",
      "        \"head\": \"a9b22a6d80c1e0bb49c1cf75a3c075b642c28f81\", \n",
      "        \"commits\": [\n",
      "            {\n",
      "                \"distinct\": true, \n",
      "                \"sha\": \"a9b22a6d80c1e0bb49c1cf75a3c075b642c28f81\", \n",
      "                \"message\": \"Altered BingBot.jar\\n\\nFixed issue with multiple account support\", \n",
      "                \"url\": \"https://api.github.com/repos/davidjhulse/davesbingrewardsbot/commits/a9b22a6d80c1e0bb49c1cf75a3c075b642c28f81\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"david.hulse@live.com\", \n",
      "                    \"name\": \"davidjhulse\"\n",
      "                }\n",
      "            }\n",
      "        ], \n",
      "        \"distinct_size\": 1, \n",
      "        \"push_id\": 536740396, \n",
      "        \"ref\": \"refs/heads/master\", \n",
      "        \"before\": \"86ffa724b4d70fce46e760f8cc080f5ec3d7d85f\"\n",
      "    }, \n",
      "    \"created_at\": \"2015-01-01T00:00:00Z\", \n",
      "    \"actor\": {\n",
      "        \"url\": \"https://api.github.com/users/davidjhulse\", \n",
      "        \"login\": \"davidjhulse\", \n",
      "        \"avatar_url\": \"https://avatars.githubusercontent.com/u/9152315?\", \n",
      "        \"id\": 9152315, \n",
      "        \"gravatar_id\": \"\"\n",
      "    }, \n",
      "    \"id\": \"2489368070\", \n",
      "    \"repo\": {\n",
      "        \"url\": \"https://api.github.com/repos/davidjhulse/davesbingrewardsbot\", \n",
      "        \"id\": 28635890, \n",
      "        \"name\": \"davidjhulse/davesbingrewardsbot\"\n",
      "    }, \n",
      "    \"type\": \"PushEvent\", \n",
      "    \"public\": true\n",
      "}\n",
      "{\n",
      "    \"payload\": {\n",
      "        \"size\": 1, \n",
      "        \"head\": \"56688cc528224d40679b7e83c105b27367443a8c\", \n",
      "        \"commits\": [\n",
      "            {\n",
      "                \"distinct\": true, \n",
      "                \"sha\": \"56688cc528224d40679b7e83c105b27367443a8c\", \n",
      "                \"message\": \"Can be run from outside of bin. Fixed multi-line issue\", \n",
      "                \"url\": \"https://api.github.com/repos/jmoon018/rshell-unit-tester/commits/56688cc528224d40679b7e83c105b27367443a8c\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"jmoon018@ucr.edu\", \n",
      "                    \"name\": \"Jamal Moon\"\n",
      "                }\n",
      "            }\n",
      "        ], \n",
      "        \"distinct_size\": 1, \n",
      "        \"push_id\": 536740397, \n",
      "        \"ref\": \"refs/heads/master\", \n",
      "        \"before\": \"d5a69a84e4d95c1c2cbc0a828172f9ce94db35c6\"\n",
      "    }, \n",
      "    \"created_at\": \"2015-01-01T00:00:00Z\", \n",
      "    \"actor\": {\n",
      "        \"url\": \"https://api.github.com/users/jmoon018\", \n",
      "        \"login\": \"jmoon018\", \n",
      "        \"avatar_url\": \"https://avatars.githubusercontent.com/u/5581438?\", \n",
      "        \"id\": 5581438, \n",
      "        \"gravatar_id\": \"\"\n",
      "    }, \n",
      "    \"id\": \"2489368072\", \n",
      "    \"repo\": {\n",
      "        \"url\": \"https://api.github.com/repos/jmoon018/rshell-unit-tester\", \n",
      "        \"id\": 26392647, \n",
      "        \"name\": \"jmoon018/rshell-unit-tester\"\n",
      "    }, \n",
      "    \"type\": \"PushEvent\", \n",
      "    \"public\": true\n",
      "}\n",
      "{\n",
      "    \"payload\": {\n",
      "        \"master_branch\": \"master\", \n",
      "        \"ref_type\": \"branch\", \n",
      "        \"ref\": \"master\", \n",
      "        \"description\": \"Making an HTML5 game from scratch.\", \n",
      "        \"pusher_type\": \"user\"\n",
      "    }, \n",
      "    \"created_at\": \"2015-01-01T00:00:01Z\", \n",
      "    \"actor\": {\n",
      "        \"url\": \"https://api.github.com/users/christoferpeterson\", \n",
      "        \"login\": \"christoferpeterson\", \n",
      "        \"avatar_url\": \"https://avatars.githubusercontent.com/u/6352424?\", \n",
      "        \"id\": 6352424, \n",
      "        \"gravatar_id\": \"\"\n",
      "    }, \n",
      "    \"id\": \"2489368089\", \n",
      "    \"repo\": {\n",
      "        \"url\": \"https://api.github.com/repos/christoferpeterson/Vadek\", \n",
      "        \"id\": 28677542, \n",
      "        \"name\": \"christoferpeterson/Vadek\"\n",
      "    }, \n",
      "    \"type\": \"CreateEvent\", \n",
      "    \"public\": true\n",
      "}\n",
      "{\n",
      "    \"payload\": {\n",
      "        \"size\": 8, \n",
      "        \"head\": \"bc9aa8a0c999232d2675cf5ba60e9c9d03f2a806\", \n",
      "        \"commits\": [\n",
      "            {\n",
      "                \"distinct\": false, \n",
      "                \"sha\": \"ec572201658b566373845657a8aca7bc6d6214a4\", \n",
      "                \"message\": \"Remove unused method.\", \n",
      "                \"url\": \"https://api.github.com/repos/square/okhttp/commits/ec572201658b566373845657a8aca7bc6d6214a4\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"jw@squareup.com\", \n",
      "                    \"name\": \"Jake Wharton\"\n",
      "                }\n",
      "            }, \n",
      "            {\n",
      "                \"distinct\": false, \n",
      "                \"sha\": \"bfa25f05bd5786c990fccb513f56375a4f98d95b\", \n",
      "                \"message\": \"Merge pull request #1262 from square/jw/unused\\n\\nRemove unused method.\", \n",
      "                \"url\": \"https://api.github.com/repos/square/okhttp/commits/bfa25f05bd5786c990fccb513f56375a4f98d95b\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"jesse@swank.ca\", \n",
      "                    \"name\": \"Jesse Wilson\"\n",
      "                }\n",
      "            }, \n",
      "            {\n",
      "                \"distinct\": false, \n",
      "                \"sha\": \"6909923b89d98fef39600a170aa50e566dbb730a\", \n",
      "                \"message\": \"Update CHANGELOG.md\", \n",
      "                \"url\": \"https://api.github.com/repos/square/okhttp/commits/6909923b89d98fef39600a170aa50e566dbb730a\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"christian.becker.1987@gmail.com\", \n",
      "                    \"name\": \"Christian Becker\"\n",
      "                }\n",
      "            }, \n",
      "            {\n",
      "                \"distinct\": false, \n",
      "                \"sha\": \"2ca6dfd055d480a4ef3520af39ad4b7ba02d5491\", \n",
      "                \"message\": \"Merge pull request #1263 from ChristianBecker/patch-1\\n\\nUpdate CHANGELOG.md\", \n",
      "                \"url\": \"https://api.github.com/repos/square/okhttp/commits/2ca6dfd055d480a4ef3520af39ad4b7ba02d5491\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"jesse@swank.ca\", \n",
      "                    \"name\": \"Jesse Wilson\"\n",
      "                }\n",
      "            }, \n",
      "            {\n",
      "                \"distinct\": false, \n",
      "                \"sha\": \"5527a1764d973d293faffe31faddf205e3f37641\", \n",
      "                \"message\": \"We now fall back to TLS 1.0.\", \n",
      "                \"url\": \"https://api.github.com/repos/square/okhttp/commits/5527a1764d973d293faffe31faddf205e3f37641\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"jwilson@squareup.com\", \n",
      "                    \"name\": \"Jesse Wilson\"\n",
      "                }\n",
      "            }, \n",
      "            {\n",
      "                \"distinct\": false, \n",
      "                \"sha\": \"31ec212f84edbe04b9b705f521ed7dc3bd7ddc3a\", \n",
      "                \"message\": \"Skip buffer allocation when not needed.\", \n",
      "                \"url\": \"https://api.github.com/repos/square/okhttp/commits/31ec212f84edbe04b9b705f521ed7dc3bd7ddc3a\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"jw@squareup.com\", \n",
      "                    \"name\": \"Jake Wharton\"\n",
      "                }\n",
      "            }, \n",
      "            {\n",
      "                \"distinct\": false, \n",
      "                \"sha\": \"7756c4e4fe1cd703cff07d218da2fecb30f916c6\", \n",
      "                \"message\": \"Merge pull request #1264 from square/jw/kill-two-allocations\\n\\nSkip buffer allocation when not needed.\", \n",
      "                \"url\": \"https://api.github.com/repos/square/okhttp/commits/7756c4e4fe1cd703cff07d218da2fecb30f916c6\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"jesse@swank.ca\", \n",
      "                    \"name\": \"Jesse Wilson\"\n",
      "                }\n",
      "            }, \n",
      "            {\n",
      "                \"distinct\": true, \n",
      "                \"sha\": \"bc9aa8a0c999232d2675cf5ba60e9c9d03f2a806\", \n",
      "                \"message\": \"Add a web socket call concept for connecting.\\n\\nSimilar to HTTP and Call, the WebSocketCall is a representation of a pending HTTP request and subsequent upgrade to speak web sockets. Upon synchronous execution you are handed a WebSocket instance for synchronous writing and also pass in a WebSocketListener for async callbacks due to reading.\\n\\nThe API changes in this commits also generalize WebSocket such that it's agnostic to being a client or server peer.\", \n",
      "                \"url\": \"https://api.github.com/repos/square/okhttp/commits/bc9aa8a0c999232d2675cf5ba60e9c9d03f2a806\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"jw@squareup.com\", \n",
      "                    \"name\": \"Jake Wharton\"\n",
      "                }\n",
      "            }\n",
      "        ], \n",
      "        \"distinct_size\": 1, \n",
      "        \"push_id\": 536740405, \n",
      "        \"ref\": \"refs/heads/jw/websocket-call\", \n",
      "        \"before\": \"8590ddf9fe54fe27d24f25dd66729993e63a5d61\"\n",
      "    }, \n",
      "    \"created_at\": \"2015-01-01T00:00:01Z\", \n",
      "    \"actor\": {\n",
      "        \"url\": \"https://api.github.com/users/JakeWharton\", \n",
      "        \"login\": \"JakeWharton\", \n",
      "        \"avatar_url\": \"https://avatars.githubusercontent.com/u/66577?\", \n",
      "        \"id\": 66577, \n",
      "        \"gravatar_id\": \"\"\n",
      "    }, \n",
      "    \"id\": \"2489368095\", \n",
      "    \"repo\": {\n",
      "        \"url\": \"https://api.github.com/repos/square/okhttp\", \n",
      "        \"id\": 5152285, \n",
      "        \"name\": \"square/okhttp\"\n",
      "    }, \n",
      "    \"org\": {\n",
      "        \"url\": \"https://api.github.com/orgs/square\", \n",
      "        \"login\": \"square\", \n",
      "        \"avatar_url\": \"https://avatars.githubusercontent.com/u/82592?\", \n",
      "        \"id\": 82592, \n",
      "        \"gravatar_id\": \"\"\n",
      "    }, \n",
      "    \"type\": \"PushEvent\", \n",
      "    \"public\": true\n",
      "}\n",
      "{\n",
      "    \"payload\": {\n",
      "        \"size\": 1, \n",
      "        \"head\": \"97a402b54bfd562b8cefd11f1361cd5af41b0535\", \n",
      "        \"commits\": [\n",
      "            {\n",
      "                \"distinct\": true, \n",
      "                \"sha\": \"97a402b54bfd562b8cefd11f1361cd5af41b0535\", \n",
      "                \"message\": \"init\", \n",
      "                \"url\": \"https://api.github.com/repos/git4ruby/movie_review1/commits/97a402b54bfd562b8cefd11f1361cd5af41b0535\", \n",
      "                \"author\": {\n",
      "                    \"email\": \"chandanamohit@gmail.com\", \n",
      "                    \"name\": \"Mohit\"\n",
      "                }\n",
      "            }\n",
      "        ], \n",
      "        \"distinct_size\": 1, \n",
      "        \"push_id\": 536740413, \n",
      "        \"ref\": \"refs/heads/master\", \n",
      "        \"before\": \"bc6e2f2a7d2095f8881d44d737506c14c1e07561\"\n",
      "    }, \n",
      "    \"created_at\": \"2015-01-01T00:00:03Z\", \n",
      "    \"actor\": {\n",
      "        \"url\": \"https://api.github.com/users/git4ruby\", \n",
      "        \"login\": \"git4ruby\", \n",
      "        \"avatar_url\": \"https://avatars.githubusercontent.com/u/9221683?\", \n",
      "        \"id\": 9221683, \n",
      "        \"gravatar_id\": \"\"\n",
      "    }, \n",
      "    \"id\": \"2489368104\", \n",
      "    \"repo\": {\n",
      "        \"url\": \"https://api.github.com/repos/git4ruby/movie_review1\", \n",
      "        \"id\": 28520835, \n",
      "        \"name\": \"git4ruby/movie_review1\"\n",
      "    }, \n",
      "    \"type\": \"PushEvent\", \n",
      "    \"public\": true\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "for event in events[:5]:\n",
    "    print json.dumps(event, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "640\n"
     ]
    }
   ],
   "source": [
    "with gzip.open(path) as f:\n",
    "    lang = [ujson.loads(line) for line in f if \"language\" in line]\n",
    "print len(lang)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F#\n",
      "Java\n",
      "PHP\n",
      "C++\n"
     ]
    }
   ],
   "source": [
    "for pr in pull_requests[:5]:\n",
    "    try:\n",
    "        if pr['payload']['pull_request']['head']['repo']['language'] is not None:\n",
    "            print pr['payload']['pull_request']['head']['repo']['language']\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>user</th>\n",
       "      <th>repo</th>\n",
       "      <th>created_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>davidjhulse</td>\n",
       "      <td>davidjhulse/davesbingrewardsbot</td>\n",
       "      <td>2015-01-01 00:00:00+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>jmoon018</td>\n",
       "      <td>jmoon018/rshell-unit-tester</td>\n",
       "      <td>2015-01-01 00:00:00+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>christoferpeterson</td>\n",
       "      <td>christoferpeterson/Vadek</td>\n",
       "      <td>2015-01-01 00:00:01+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>JakeWharton</td>\n",
       "      <td>square/okhttp</td>\n",
       "      <td>2015-01-01 00:00:01+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>git4ruby</td>\n",
       "      <td>git4ruby/movie_review1</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>tlgkccampbell</td>\n",
       "      <td>tlgkccampbell/ultraviolet</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>Vilyan01</td>\n",
       "      <td>Vilyan01/ILP</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>xndcn</td>\n",
       "      <td>xndcn/d-statistics</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>team3cord</td>\n",
       "      <td>team3cord/mc-dotfiles</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>greyia</td>\n",
       "      <td>greyia/port2container</td>\n",
       "      <td>2015-01-01 00:00:04+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          type                user                             repo  \\\n",
       "0    PushEvent         davidjhulse  davidjhulse/davesbingrewardsbot   \n",
       "1    PushEvent            jmoon018      jmoon018/rshell-unit-tester   \n",
       "2  CreateEvent  christoferpeterson         christoferpeterson/Vadek   \n",
       "3    PushEvent         JakeWharton                    square/okhttp   \n",
       "4    PushEvent            git4ruby           git4ruby/movie_review1   \n",
       "5    PushEvent       tlgkccampbell        tlgkccampbell/ultraviolet   \n",
       "6    PushEvent            Vilyan01                     Vilyan01/ILP   \n",
       "7    PushEvent               xndcn               xndcn/d-statistics   \n",
       "8    PushEvent           team3cord            team3cord/mc-dotfiles   \n",
       "9  CreateEvent              greyia            greyia/port2container   \n",
       "\n",
       "                  created_at  \n",
       "0  2015-01-01 00:00:00+00:00  \n",
       "1  2015-01-01 00:00:00+00:00  \n",
       "2  2015-01-01 00:00:01+00:00  \n",
       "3  2015-01-01 00:00:01+00:00  \n",
       "4  2015-01-01 00:00:03+00:00  \n",
       "5  2015-01-01 00:00:03+00:00  \n",
       "6  2015-01-01 00:00:03+00:00  \n",
       "7  2015-01-01 00:00:03+00:00  \n",
       "8  2015-01-01 00:00:03+00:00  \n",
       "9  2015-01-01 00:00:04+00:00  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame.from_records(((event['type'], event['actor']['login'], event['repo']['name'], pd.Timestamp(event['created_at'])) \n",
    "                          for event in events[:100]),\n",
    "                          columns=['type', 'user', 'repo', 'created_at'])\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>user</th>\n",
       "      <th>repo</th>\n",
       "      <th>created_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>davidjhulse</td>\n",
       "      <td>davidjhulse/davesbingrewardsbot</td>\n",
       "      <td>2015-01-01 00:00:00+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>jmoon018</td>\n",
       "      <td>jmoon018/rshell-unit-tester</td>\n",
       "      <td>2015-01-01 00:00:00+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>christoferpeterson</td>\n",
       "      <td>christoferpeterson/Vadek</td>\n",
       "      <td>2015-01-01 00:00:01+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>JakeWharton</td>\n",
       "      <td>square/okhttp</td>\n",
       "      <td>2015-01-01 00:00:01+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>git4ruby</td>\n",
       "      <td>git4ruby/movie_review1</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>tlgkccampbell</td>\n",
       "      <td>tlgkccampbell/ultraviolet</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>Vilyan01</td>\n",
       "      <td>Vilyan01/ILP</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>xndcn</td>\n",
       "      <td>xndcn/d-statistics</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>team3cord</td>\n",
       "      <td>team3cord/mc-dotfiles</td>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>greyia</td>\n",
       "      <td>greyia/port2container</td>\n",
       "      <td>2015-01-01 00:00:04+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>slurp-logs</td>\n",
       "      <td>piscisaureus/slurp-logs</td>\n",
       "      <td>2015-01-01 00:00:05+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "           type                user                             repo  \\\n",
       "0     PushEvent         davidjhulse  davidjhulse/davesbingrewardsbot   \n",
       "1     PushEvent            jmoon018      jmoon018/rshell-unit-tester   \n",
       "2   CreateEvent  christoferpeterson         christoferpeterson/Vadek   \n",
       "3     PushEvent         JakeWharton                    square/okhttp   \n",
       "4     PushEvent            git4ruby           git4ruby/movie_review1   \n",
       "5     PushEvent       tlgkccampbell        tlgkccampbell/ultraviolet   \n",
       "6     PushEvent            Vilyan01                     Vilyan01/ILP   \n",
       "7     PushEvent               xndcn               xndcn/d-statistics   \n",
       "8     PushEvent           team3cord            team3cord/mc-dotfiles   \n",
       "9   CreateEvent              greyia            greyia/port2container   \n",
       "10    PushEvent          slurp-logs          piscisaureus/slurp-logs   \n",
       "\n",
       "                   created_at  \n",
       "0   2015-01-01 00:00:00+00:00  \n",
       "1   2015-01-01 00:00:00+00:00  \n",
       "2   2015-01-01 00:00:01+00:00  \n",
       "3   2015-01-01 00:00:01+00:00  \n",
       "4   2015-01-01 00:00:03+00:00  \n",
       "5   2015-01-01 00:00:03+00:00  \n",
       "6   2015-01-01 00:00:03+00:00  \n",
       "7   2015-01-01 00:00:03+00:00  \n",
       "8   2015-01-01 00:00:03+00:00  \n",
       "9   2015-01-01 00:00:04+00:00  \n",
       "..."
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = bz.Data(df)\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bz.compute(ds[ds.type=='CreateEvent'].user.distinct().count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>user</th>\n",
       "      <th>repo</th>\n",
       "      <th>created_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>jchristi</td>\n",
       "      <td>LinuxStandardBase/lsb</td>\n",
       "      <td>2015-01-01 00:00:18+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>tategakibunko</td>\n",
       "      <td>inf0rmer/blanket</td>\n",
       "      <td>2015-01-01 00:00:24+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>alfateam123</td>\n",
       "      <td>parrt/cs652</td>\n",
       "      <td>2015-01-01 00:00:35+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          type           user                   repo  \\\n",
       "65  WatchEvent       jchristi  LinuxStandardBase/lsb   \n",
       "72  WatchEvent  tategakibunko       inf0rmer/blanket   \n",
       "91  WatchEvent    alfateam123            parrt/cs652   \n",
       "\n",
       "                   created_at  \n",
       "65  2015-01-01 00:00:18+00:00  \n",
       "72  2015-01-01 00:00:24+00:00  \n",
       "91  2015-01-01 00:00:35+00:00  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.type=='WatchEvent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>repo</th>\n",
       "      <th>total</th>\n",
       "      <th>user</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>56</td>\n",
       "      <td>64</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "                            type  repo  total  user\n",
       "0                    CreateEvent    12     12    12\n",
       "1                    DeleteEvent     1      1     1\n",
       "2                      ForkEvent     1      1     1\n",
       "3              IssueCommentEvent    14     14    13\n",
       "4                    IssuesEvent     2      2     1\n",
       "5               PullRequestEvent     2      2     2\n",
       "6  PullRequestReviewCommentEvent     1      1     1\n",
       "7                      PushEvent    56     64    54\n",
       "8                     WatchEvent     2      3     2"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bz.by(ds.type, total=ds.type.count(), user=ds.user.distinct().count(), repo=ds.repo.distinct().count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>repo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CreateEvent</th>\n",
       "      <td>9</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DeleteEvent</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ForkEvent</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IssueCommentEvent</th>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IssuesEvent</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PullRequestEvent</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PullRequestReviewCommentEvent</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PushEvent</th>\n",
       "      <td>60</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WatchEvent</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               user  repo\n",
       "type                                     \n",
       "CreateEvent                       9    12\n",
       "DeleteEvent                       1     1\n",
       "ForkEvent                         1     1\n",
       "IssueCommentEvent                14    14\n",
       "IssuesEvent                       2     2\n",
       "PullRequestEvent                  2     2\n",
       "PullRequestReviewCommentEvent     1     1\n",
       "PushEvent                        60    64\n",
       "WatchEvent                        3     3"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('type')[['user', 'repo']].agg(lambda x: x.drop_duplicates().count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>repo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>CreateEvent</th>\n",
       "      <td>9</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DeleteEvent</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ForkEvent</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IssueCommentEvent</th>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IssuesEvent</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PullRequestEvent</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PullRequestReviewCommentEvent</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PushEvent</th>\n",
       "      <td>60</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WatchEvent</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               user  repo\n",
       "type                                     \n",
       "CreateEvent                       9    12\n",
       "DeleteEvent                       1     1\n",
       "ForkEvent                         1     1\n",
       "IssueCommentEvent                14    14\n",
       "IssuesEvent                       2     2\n",
       "PullRequestEvent                  2     2\n",
       "PullRequestReviewCommentEvent     1     1\n",
       "PushEvent                        60    64\n",
       "WatchEvent                        3     3"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('type')[['user', 'repo']].agg(lambda x: len(np.unique(sorted(x))))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}