algorithms/07 HP Scraper.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:203b8eb83e9dbadb0ebce9eea6787308a638b4f4e9f04a2564e0bb643f957060"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "First let's test out the scraping"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from bs4 import BeautifulSoup\n",
      "import urllib2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "url = \"https://www.fanfiction.net/book/Harry-Potter/?&srt=1&lan=1&r=10&p=1\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# First pull the search page\n",
      "page = urllib2.urlopen(url)\n",
      "soup = BeautifulSoup(page.read())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Look at the links (peeked at the HTML using web inspector)\n",
      "links = soup.findAll('a',{'class':'stitle'})\n",
      "links[:2]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# See what the links look like\n",
      "links[0]['href']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Form the full urls from the links\n",
      "urls = [\"https://www.fanfiction.net\" + link['href'] for link in links]\n",
      "urls[:10]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Looks fine. Now try it by paginating across\n",
      "all_urls = []\n",
      "# Note: range(x,y) is inclusive of the first number, exclusive of the second number \n",
      "# So range(1,3) will include 1 and 2 but not 3\n",
      "for i in range(1,26):\n",
      "    print \"Page \" + str(i) + \", up to \" + str(len(all_urls))\n",
      "    url = \"https://www.fanfiction.net/book/Harry-Potter/?&srt=1&lan=1&r=10&p=\" + str(i)\n",
      "    page = urllib2.urlopen(url)\n",
      "    soup = BeautifulSoup(page.read())\n",
      "    links = soup.findAll('a',{'class':'stitle'})\n",
      "    urls = [\"https://www.fanfiction.net\" + link['href'] for link in links]\n",
      "    # to add an array's elements to another array, use .extend instead of .append\n",
      "    all_urls.extend(urls)\n",
      "len(all_urls)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now let's take a look at a single page\n",
      "url = all_urls[0]\n",
      "page = urllib2.urlopen(url)\n",
      "soup = BeautifulSoup(page.read())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "div = soup.find('div',{'class':'storytext'})\n",
      "text = div.get_text()\n",
      "text = text.encode(\"ascii\",\"ignore\")\n",
      "text[:500]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now let's write it to a file\n",
      "story_id = url.split(\"/\")[4]\n",
      "filename = \"hp/\" + story_id + \".txt\"    \n",
      "with open(filename, 'w') as f:\n",
      "    f.write(text)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# That was pretty easy. Let's grab all of them and save them to\n",
      "# A file named after their story id from the URL\n",
      "all_texts = []\n",
      "for url in all_urls:\n",
      "    story_id = url.split(\"/\")[4]\n",
      "    filename = \"hp/\" + story_id + \".txt\"\n",
      "    page = urllib2.urlopen(url)\n",
      "    soup = BeautifulSoup(page.read())\n",
      "    div = soup.find('div',{'class':'storytext'})\n",
      "    text = div.get_text()\n",
      "    text = text.encode(\"ascii\",\"ignore\")\n",
      "    with open(filename, 'w') as f:\n",
      "        f.write(text)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}