notebooks/Day_10_requests_lxml.ipynb

{
 "metadata": {
  "name": "Day_10_requests_lxml"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Making http requests in Python"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "a refresher on http might be in order: http://mashupguide.net/1.0/html/ch06s04.xhtml#d0e10474"
     ]
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "requests library"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "I like using requests even though there are several alternatives: http://lumberjaph.net/python/2012/02/17/HTTP_requests_with_python.html\n",
      "\n",
      "\n",
      "http://docs.python-requests.org/en/latest/\n",
      "\n",
      "enpkg knows about requests:\n",
      "\n",
      "> enpkg -s \n",
      "\n",
      "yields\n",
      "\n",
      "    Name                   Versions           Note\n",
      "    ------------------------------------------------------------\n",
      "    requests               0.3.0-1            \n",
      "                           0.3.1-1            \n",
      "                           0.3.2-1            \n",
      "                           0.4.1-1            \n",
      "                           0.5.0-1            \n",
      "                           0.6.1-1            \n",
      "                           0.6.4-1            \n",
      "                           0.7.4-1            \n",
      "                           0.9.0-1            \n",
      "                           0.9.1-1            \n",
      "                           0.9.3-1            \n",
      "                           0.10.1-1           \n",
      "\n",
      "\n"
     ]
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "geocoding"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "normally, I like to use geocoder.us as an example -- see my book http://mashupguide.net/1.0/html/ch13s06.xhtml#d0e21349  -- but there have been overuse by bad actors"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import requests\n",
      "import json\n",
      "\n",
      "\n",
      "# https://developers.google.com/maps/documentation/geocoding/\n",
      "# get Google lat, long\n",
      "url = \"http://maps.googleapis.com/maps/api/geocode/json?address=1600+Amphitheatre+Parkway,+Mountain+View,+CA&sensor=false\"\n",
      "\n",
      "r = requests.get(url)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "<Response [200]>"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "r.status_code"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "200"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "r.headers['content-type']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "'application/json; charset=UTF-8'"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "r.text"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "u'{\\n   \"results\" : [\\n      {\\n         \"address_components\" : [\\n            {\\n               \"long_name\" : \"1600\",\\n               \"short_name\" : \"1600\",\\n               \"types\" : [ \"street_number\" ]\\n            },\\n            {\\n               \"long_name\" : \"Amphitheatre Parkway\",\\n               \"short_name\" : \"Amphitheatre Pkwy\",\\n               \"types\" : [ \"route\" ]\\n            },\\n            {\\n               \"long_name\" : \"Mountain View\",\\n               \"short_name\" : \"Mountain View\",\\n               \"types\" : [ \"locality\", \"political\" ]\\n            },\\n            {\\n               \"long_name\" : \"Santa Clara\",\\n               \"short_name\" : \"Santa Clara\",\\n               \"types\" : [ \"administrative_area_level_2\", \"political\" ]\\n            },\\n            {\\n               \"long_name\" : \"California\",\\n               \"short_name\" : \"CA\",\\n               \"types\" : [ \"administrative_area_level_1\", \"political\" ]\\n            },\\n            {\\n               \"long_name\" : \"United States\",\\n               \"short_name\" : \"US\",\\n               \"types\" : [ \"country\", \"political\" ]\\n            },\\n            {\\n               \"long_name\" : \"94043\",\\n               \"short_name\" : \"94043\",\\n               \"types\" : [ \"postal_code\" ]\\n            }\\n         ],\\n         \"formatted_address\" : \"1600 Amphitheatre Parkway, Mountain View, CA 94043, USA\",\\n         \"geometry\" : {\\n            \"location\" : {\\n               \"lat\" : 37.42219410,\\n               \"lng\" : -122.08459320\\n            },\\n            \"location_type\" : \"ROOFTOP\",\\n            \"viewport\" : {\\n               \"northeast\" : {\\n                  \"lat\" : 37.42354308029149,\\n                  \"lng\" : -122.0832442197085\\n               },\\n               \"southwest\" : {\\n                  \"lat\" : 37.42084511970850,\\n                  \"lng\" : -122.0859421802915\\n               }\\n            }\\n         },\\n         \"types\" : [ \"street_address\" ]\\n      }\\n   ],\\n   \"status\" : \"OK\"\\n}\\n'"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "r.json()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "{u'results': [{u'address_components': [{u'long_name': u'1600',\n",
        "     u'short_name': u'1600',\n",
        "     u'types': [u'street_number']},\n",
        "    {u'long_name': u'Amphitheatre Parkway',\n",
        "     u'short_name': u'Amphitheatre Pkwy',\n",
        "     u'types': [u'route']},\n",
        "    {u'long_name': u'Mountain View',\n",
        "     u'short_name': u'Mountain View',\n",
        "     u'types': [u'locality', u'political']},\n",
        "    {u'long_name': u'Santa Clara',\n",
        "     u'short_name': u'Santa Clara',\n",
        "     u'types': [u'administrative_area_level_2', u'political']},\n",
        "    {u'long_name': u'California',\n",
        "     u'short_name': u'CA',\n",
        "     u'types': [u'administrative_area_level_1', u'political']},\n",
        "    {u'long_name': u'United States',\n",
        "     u'short_name': u'US',\n",
        "     u'types': [u'country', u'political']},\n",
        "    {u'long_name': u'94043',\n",
        "     u'short_name': u'94043',\n",
        "     u'types': [u'postal_code']}],\n",
        "   u'formatted_address': u'1600 Amphitheatre Parkway, Mountain View, CA 94043, USA',\n",
        "   u'geometry': {u'location': {u'lat': 37.4221941, u'lng': -122.0845932},\n",
        "    u'location_type': u'ROOFTOP',\n",
        "    u'viewport': {u'northeast': {u'lat': 37.42354308029149,\n",
        "      u'lng': -122.0832442197085},\n",
        "     u'southwest': {u'lat': 37.4208451197085, u'lng': -122.0859421802915}}},\n",
        "   u'types': [u'street_address']}],\n",
        " u'status': u'OK'}"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "r.json()['results'][0]['geometry']['location']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 18,
       "text": [
        "{u'lat': 37.4221941, u'lng': -122.0845932}"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "lxml for a bit of scraping"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import requests\n",
      "from lxml.html import parse\n",
      "from StringIO import StringIO\n",
      "\n",
      "ry_class_url = \"http://osoc.berkeley.edu/OSOC/osoc?p_term=SP&p_deptname=INFO&p_instr=yee\"\n",
      "r = requests.get(ry_class_url)\n",
      "doc = parse(StringIO(r.content)).getroot()\n",
      "\n",
      "course_tts = doc.cssselect('table:nth-of-type(2) tt')\n",
      "\n",
      "print course_tts[-1].text_content().replace(\"Avail Seats\", \"Avail_Seats\").split(\" \")[:-1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['Limit:40', 'Enrolled:31', 'Waitlist:0', 'Avail_Seats:9']\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}