EMSE6992_Labs/lab_webscraping/Lab_2_B_Live.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'pattern'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-ccf3998ef2d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Setup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpattern\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweb\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mweb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pattern'"
     ]
    }
   ],
   "source": [
    "# Setup\n",
    "import pattern.web as web\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from cs109style import customize_mpl, customize_css\n",
    "customize_mpl()\n",
    "customize_css()\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Example 2: extracting reddit titles, upvotes, downvotes, and submission time\n",
    "\n",
    "### We'll operate in two phases:\n",
    "* first, find all the URLs to comment pages on the first few front pages of reddit.\n",
    "* second, extract information from each comments page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_links_from_front_pages(n):\n",
    "    'find  URLs of comments pages, linked from the n first few pages of reddit'\n",
    "    url = web.URL('http://www.reddit.com/')\n",
    "    comment_pages = []\n",
    "    for page_idx in range(n):\n",
    "        dom = web.DOM(url.download(cached=False))\n",
    "        \n",
    "        ### Extract comments pages\n",
    "        \n",
    "        ### find the next page link - reddit has 25 links per page\n",
    "\n",
    "    # use set() to remove repeated URLs\n",
    "    return list(set(comment_pages))\n",
    "\n",
    "            \n",
    "print len(get_links_from_front_pages(6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def info_from_comments_pages(links):\n",
    "    'fetch title, upvotes, downvotes, time of submission from a sequence of links'\n",
    "    results = []\n",
    "    for urltext in links:\n",
    "        url = web.URL(urltext)\n",
    "        print \"fetching info for\", url\n",
    "        try:\n",
    "            dom = web.DOM(url.download(cached=False))\n",
    "            \n",
    "            ### Extract title, upvotes, downvotes, submission time\n",
    "            \n",
    "            results.append((title, upvotes, downvotes, pd.to_datetime(time)))\n",
    "        except KeyboardInterrupt:\n",
    "            # allow us to interrupt the kernel and still continue\n",
    "            break\n",
    "        except:\n",
    "            pass  # some things that look like comment pages don't have the information above\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "comments_pages = get_links_from_front_pages(5)\n",
    "print \"Fetching info for\", len(comments_pages), \"pages\"\n",
    "pages = info_from_comments_pages(comments_pages)\n",
    "titles, upvotes, downvotes, dates = zip(*pages)  # zip(*seq) transposes a sequence of sequences.\n",
    "df = pd.DataFrame({'title' : titles, 'upvotes' : upvotes, 'downvotes' : downvotes, 'date' : dates}, index=dates)\n",
    "print df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sort('date', inplace=True)\n",
    "df['upvotes'].plot(c='g')\n",
    "df['downvotes'].plot(c='r')\n",
    "(df['upvotes'] - df['downvotes']).plot(c='k')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}