Consuming APIs.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:422e232381e881c25ffdf8c4eb8c08431b88cda7371cf59491b64f96934f0625"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#Consuming APIs with requests\n",
      "\n",
      "We will look at Hacker New's API as it does not require authentication!"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import requests\n",
      "import pandas as pd\n",
      "import json"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 46
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "top_100_stories = requests.get('https://hacker-news.firebaseio.com/v0/topstories.json')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 28
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "top_100_stories is a string, not a list!"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print(top_100_stories.text)\n",
      "print(type(top_100_stories.text))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[8988819,8987434,8989455,8988949,8985625,8986920,8988984,8987441,8986551,8986183,8984648,8983355,8989495,8985151,8987931,8987014,8987459,8984266,8989453,8988594,8986497,8987602,8989245,8988902,8984343,8984797,8983771,8986459,8985179,8985496,8989138,8988301,8987813,8985541,8985217,8984508,8983804,8987783,8985376,8988837,8987676,8983158,8985211,8985349,8986998,8985976,8984425,8988403,8982526,8984385,8986489,8983875,8984493,8983441,8985134,8988612,8984580,8988279,8982962,8984129,8988684,8987727,8988332,8986000,8986127,8983938,8987698,8986624,8984926,8985334,8979620,8987380,8984497,8981690,8985009,8987855,8985124,8989110,8987550,8981642,8980255,8988291,8980498,8983905,8982013,8981314,8981707,8988248,8983973,8986395,8984173,8982620,8977550,8988389,8982241,8983134,8987027,8986575,8989093,8986547]\n",
        "<class 'str'>\n"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "json.loads the string!"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "top_100_stories_list = json.loads(top_100_stories.text)\n",
      "print(type(top_100_stories_list))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<class 'list'>\n"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "top_100_story_json_list = []\n",
      "\n",
      "for story in top_100_stories_list:\n",
      "    story_request = requests.get('https://hacker-news.firebaseio.com/v0/item/'+str(story)+'.json')\n",
      "    top_100_story_json_list.append(json.loads(story_request.text))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "with open(r'./Assets/top_100_hacker_news_stories.json', 'w') as outfile:\n",
      "    json.dump(top_100_story_json_list, outfile)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 48
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df = pd.read_json(r'./Assets/top_100_hacker_news_stories.json')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 49
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "The time value is in Unix Time (seconds since epoch) - we change this to date-time"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df['date_time'] = pd.to_datetime(df['time'], unit='s')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 57
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.sort('score', ascending=False)[['score', 'title']][0:5]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>score</th>\n",
        "      <th>title</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
        "      <td> 686</td>\n",
        "      <td>                     Windows 10 for Raspberry Pi 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>48</th>\n",
        "      <td> 564</td>\n",
        "      <td>             Turbocharged Raspberry Pi 2 unleashed</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4 </th>\n",
        "      <td> 540</td>\n",
        "      <td> F.C.C. Is Expected to Propose Regulating the I...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1 </th>\n",
        "      <td> 427</td>\n",
        "      <td>      Google Is Developing Its Own Uber Competitor</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
        "      <td> 356</td>\n",
        "      <td>                      What Color Is Your Function?</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 70,
       "text": [
        "    score                                              title\n",
        "11    686                      Windows 10 for Raspberry Pi 2\n",
        "48    564              Turbocharged Raspberry Pi 2 unleashed\n",
        "4     540  F.C.C. Is Expected to Propose Regulating the I...\n",
        "1     427       Google Is Developing Its Own Uber Competitor\n",
        "10    356                       What Color Is Your Function?"
       ]
      }
     ],
     "prompt_number": 70
    }
   ],
   "metadata": {}
  }
 ]
}