code/04.PythonCrawlerGovernmentReport.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "\n",
    "\n",
    "***\n",
    "***\n",
    "# 数据抓取：\n",
    "   > # 抓取47年政府工作报告\n",
    "***\n",
    "***\n",
    "\n",
    "王成军 \n",
    "\n",
    "wangchengjun@nju.edu.cn\n",
    "\n",
    "计算传播网 http://computational-communication.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "import urllib2\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false,
    "scrolled": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe src=http://www.hprc.org.cn/wxzl/wxysl/lczf/ width=1000 height=500></iframe>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import display_html, HTML\n",
    "HTML('<iframe src=http://www.hprc.org.cn/wxzl/wxysl/lczf/ width=1000 height=500></iframe>')\n",
    "# the webpage we would like to crawl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspect\n",
    "\n",
    "# <td width=\"274\" class=\"bl\">·&nbsp;<a href=\"./d12qgrdzfbg/201603/t20160318_369509.html\" target=\"_blank\" title=\"2016年政府工作报告\">2016年政府工作报告</a></td>\n",
    "\n",
    "    <td width=\"274\" class=\"bl\">·&nbsp;<a href=\"./d12qgrdzfbg/201603/t20160318_369509.html\" target=\"_blank\" title=\"2016年政府工作报告\">2016年政府工作报告</a></td>\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'./d12qgrdzfbg/201603/t20160318_369509.html'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the link for each year\n",
    "url = \"http://www.hprc.org.cn/wxzl/wxysl/lczf/\" \n",
    "content = urllib2.urlopen(url).read().decode('gb18030') \n",
    "soup = BeautifulSoup(content, 'html.parser') \n",
    "# links = soup.find_all('td', {'class', 'bl'})   \n",
    "links = soup.select('.bl a')\n",
    "links[0]['href']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# decode\n",
    "urllib2.urlopen(url).read().decode('gb18030') \n",
    "\n",
    "# html.parser\n",
    "BeautifulSoup(content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47\n"
     ]
    }
   ],
   "source": [
    "print len(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./d12qgrdzfbg/201603/t20160318_369509.html\n"
     ]
    }
   ],
   "source": [
    "print links[0]['href']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"./d12qgrdzfbg/201603/t20160318_369509.html\" target=\"_blank\" title=\"2016年政府工作报告\">2016年政府工作报告</a>\n"
     ]
    }
   ],
   "source": [
    "print links[0].a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./d12qgrdzfbg/201603/t20160318_369509.html\n"
     ]
    }
   ],
   "source": [
    "print links[0].a['href']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d12qgrdzfbg/201603/t20160318_369509.html\n"
     ]
    }
   ],
   "source": [
    "print links[0]['href'].split('./')[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d12qgrdzfbg/201603/t20160318_369509.html\n"
     ]
    }
   ],
   "source": [
    "print links[0]['href'].split('./')[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201603/t20160318_369509.html\n"
     ]
    }
   ],
   "source": [
    "print url + links[0]['href'].split('./')[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201603/t20160318_369509.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201503/t20150318_319434.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201403/t20140315_270863.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201402/t20140214_266528.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie/201402/t20140214_266527.html']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyperlinks = [url + i['href'].split('./')[1] for i in links]\n",
    "hyperlinks[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201603/t20160318_369509.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201503/t20150318_319434.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201403/t20140315_270863.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/d12qgrdzfbg/201402/t20140214_266528.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie/201402/t20140214_266527.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie/201103/t20110315_153641.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie/201003/t20100315_44772.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie/200908/t20090817_27504.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie/200908/t20090817_27495.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_27775.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_27765.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_27757.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_27756.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_27753.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_2/200908/t20090818_27744.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_2/200908/t20090818_27741.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_2/200908/t20090818_27738.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_2/200908/t20090818_27737.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_2/200908/t20090818_27736.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_3/200908/t20090818_27709.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_3/200908/t20090818_27708.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_3/200908/t20090818_27707.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_3/200908/t20090818_27706.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_3/200908/t20090818_27705.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_4/200908/t20090818_27702.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_4/200908/t20090818_27700.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_4/200908/t20090818_27699.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_4/200908/t20090818_27678.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_4/200908/t20090818_27644.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_5/200908/t20090818_27642.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_5/200908/t20090818_27640.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_5/200908/t20090818_27616.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_5/200908/t20090818_27615.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_5/200908/t20090818_27614.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_6/200908/t20090818_27613.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_6/200908/t20090818_27612.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_6/200908/t20090818_27611.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_6/200908/t20090818_27567.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_6/200908/t20090818_27566.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_7/200908/t20090818_27565.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_8/200908/t20090818_27564.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_9/200908/t20090818_27562.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_9/200908/t20090818_27563.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_10/200908/t20090818_27561.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_10/200908/t20090818_27560.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_10/200908/t20090818_27559.html',\n",
       " u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_10/200908/t20090818_27558.html']"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyperlinks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_27775.html'"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyperlinks[9] # 2007年有分页"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe src=http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_27775.html width=1000 height=500></iframe>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import display_html, HTML\n",
    "HTML('<iframe src=http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_27775.html \\\n",
    "width=1000 height=500></iframe>')\n",
    "# 2007年有分页"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "# Inspect 下一页\n",
    "\n",
    "<a href=\"t20090818_27775_1.html\"><span style=\"color:#0033FF;font-weight:bold\">下一页</span></a>\n",
    "\n",
    "    <a href=\"t20090818_27775_1.html\"><span style=\"color:#0033FF;font-weight:bold\">下一页</span></a>\n",
    "    \n",
    "- a\n",
    "    - script\n",
    "        - td"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "url_i = 'http://www.hprc.org.cn/wxzl/wxysl/lczf/dishiyijie_1/200908/t20090818_27775.html'\n",
    "content = urllib2.urlopen(url_i).read().decode('gb18030')  \n",
    "soup = BeautifulSoup(content, 'html.parser') \n",
    "#scripts = soup.find_all('script')\n",
    "#scripts[0]\n",
    "\n",
    "scripts = soup.select('td script')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<script>\\n\\tvar currentPage = 0;//\\u6240\\u5728\\u9875\\u4ece0\\u5f00\\u59cb\\n\\tvar prevPage = currentPage-1//\\u4e0a\\u4e00\\u9875\\n\\tvar \\u4e0b\\u4e00\\u9875Page = currentPage+1//\\u4e0b\\u4e00\\u9875\\n\\tvar countPage = 4//\\u5171\\u591a\\u5c11\\u9875\\n\\t//document.write(\"\\u5171\"+countPage+\"\\u9875&nbsp;&nbsp;\");\\n\\t\\n\\t//\\u5faa\\u73af\\n\\tvar num = 17;\\n\\tfor(var i=0+(currentPage-1-(currentPage-1)%num) ; i<=(num+(currentPage-1-(currentPage-1)%num))&&(i<countPage) ; i++){\\n\\t\\tif(countPage >1){\\n\\t\\t\\tif(currentPage==i)\\n\\t\\t\\t\\tdocument.write(\"\\u3010<span style=\\\\\"color:#FF0000;\\\\\" class=\\\\\"hui14_30_h\\\\\">\"+(i+1)+\"</span>\\u3011&nbsp;\");\\n\\t\\t\\telse if(i==0)\\n\\t\\t\\t\\tdocument.write(\"<a href=\\\\\"t20090818_27775.html\\\\\" class=\\\\\"hui14_30_h\\\\\">\\u3010\"+(i+1)+\"\\u3011</a>&nbsp;\");\\n\\t\\t\\telse\\n\\t\\t\\t\\tdocument.write(\"<a href=\\\\\"t20090818_27775\"+\"_\" + i + \".\"+\"html\\\\\" class=\\\\\"hui14_30_h\\\\\">\\u3010\"+(i+1)+\"\\u3011</a>&nbsp;\");\\n\\t\\t}\\t\\n\\t}\\n\\t\\n\\tdocument.write(\"<br><br>\");\\n\\t//\\u8bbe\\u7f6e\\u4e0a\\u4e00\\u9875\\u4ee3\\u7801\\n\\tif(countPage>1&&currentPage!=0&&currentPage!=1)\\n\\t\\tdocument.write(\"<a href=\\\\\"t20090818_27775\"+\"_\" + prevPage + \".\"+\"html\\\\\"><span style=\\\\\"color:#0033FF;font-weight:bold\\\\\">\\u4e0a\\u4e00\\u9875</span></a>&nbsp;\");\\n\\telse if(countPage>1&&currentPage!=0&&currentPage==1)\\n\\t\\tdocument.write(\"<a href=\\\\\"t20090818_27775.html\\\\\"><span style=\\\\\"color:#0033FF;font-weight:bold\\\\\">\\u4e0a\\u4e00\\u9875</span></a>&nbsp;\");\\n\\t//else\\n\\t//\\tdocument.write(\"\\u4e0a\\u4e00\\u9875 &nbsp;\");\\n\\t\\n\\t\\n\\t//\\u8bbe\\u7f6e\\u4e0b\\u4e00\\u9875\\u4ee3\\u7801 \\n\\tif(countPage>1&&currentPage!=(countPage-1))\\n\\t\\tdocument.write(\"<a href=\\\\\"t20090818_27775\"+\"_\" + \\u4e0b\\u4e00\\u9875Page + \".\"+\"html\\\\\" ><span style=\\\\\"color:#0033FF;font-weight:bold\\\\\">\\u4e0b\\u4e00\\u9875</span></a> &nbsp;\");\\n\\t//else\\n\\t//\\tdocument.write(\"\\u4e0b\\u4e00\\u9875 &nbsp;\");\\n\\t\\t\\t\\t\\t \\n\\t</script>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scripts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\tvar currentPage = 0;//所在页从0开始\n",
      "\tvar prevPage = currentPage-1//上一页\n",
      "\tvar 下一页Page = currentPage+1//下一页\n",
      "\tvar countPage = 4//共多少页\n",
      "\t//document.write(\"共\"+countPage+\"页&nbsp;&nbsp;\");\n",
      "\t\n",
      "\t//循环\n",
      "\tvar num = 17;\n",
      "\tfor(var i=0+(currentPage-1-(currentPage-1)%num) ; i<=(num+(currentPage-1-(currentPage-1)%num))&&(i<countPage) ; i++){\n",
      "\t\tif(countPage >1){\n",
      "\t\t\tif(currentPage==i)\n",
      "\t\t\t\tdocument.write(\"【<span style=\\\"color:#FF0000;\\\" class=\\\"hui14_30_h\\\">\"+(i+1)+\"</span>】&nbsp;\");\n",
      "\t\t\telse if(i==0)\n",
      "\t\t\t\tdocument.write(\"<a href=\\\"t20090818_27775.html\\\" class=\\\"hui14_30_h\\\">【\"+(i+1)+\"】</a>&nbsp;\");\n",
      "\t\t\telse\n",
      "\t\t\t\tdocument.write(\"<a href=\\\"t20090818_27775\"+\"_\" + i + \".\"+\"html\\\" class=\\\"hui14_30_h\\\">【\"+(i+1)+\"】</a>&nbsp;\");\n",
      "\t\t}\t\n",
      "\t}\n",
      "\t\n",
      "\tdocument.write(\"<br><br>\");\n",
      "\t//设置上一页代码\n",
      "\tif(countPage>1&&currentPage!=0&&currentPage!=1)\n",
      "\t\tdocument.write(\"<a href=\\\"t20090818_27775\"+\"_\" + prevPage + \".\"+\"html\\\"><span style=\\\"color:#0033FF;font-weight:bold\\\">上一页</span></a>&nbsp;\");\n",
      "\telse if(countPage>1&&currentPage!=0&&currentPage==1)\n",
      "\t\tdocument.write(\"<a href=\\\"t20090818_27775.html\\\"><span style=\\\"color:#0033FF;font-weight:bold\\\">上一页</span></a>&nbsp;\");\n",
      "\t//else\n",
      "\t//\tdocument.write(\"上一页 &nbsp;\");\n",
      "\t\n",
      "\t\n",
      "\t//设置下一页代码 \n",
      "\tif(countPage>1&&currentPage!=(countPage-1))\n",
      "\t\tdocument.write(\"<a href=\\\"t20090818_27775\"+\"_\" + 下一页Page + \".\"+\"html\\\" ><span style=\\\"color:#0033FF;font-weight:bold\\\">下一页</span></a> &nbsp;\");\n",
      "\t//else\n",
      "\t//\tdocument.write(\"下一页 &nbsp;\");\n",
      "\t\t\t\t\t \n",
      "\t\n"
     ]
    }
   ],
   "source": [
    "print scripts.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countPage = int(''.join(scripts).split('countPage = ')[1].split('//')[0])\n",
    "countPage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "def crawler(url_i):\n",
    "    content = urllib2.urlopen(url_i).read().decode('gb18030')  \n",
    "    soup = BeautifulSoup(content, 'html.parser') \n",
    "    year = soup.find('span', {'class', 'huang16c'}).text[:4]\n",
    "    year = int(year)\n",
    "    report = ''.join(s.text for s in soup('p'))\n",
    "    # 找到分页信息\n",
    "    scripts = soup.find_all('script')\n",
    "    countPage = int(''.join(scripts[1]).split('countPage = ')[1].split('//')[0])\n",
    "    if countPage == 1:\n",
    "        pass\n",
    "    else:\n",
    "        for i in range(1, countPage):\n",
    "            url_child = url_i.split('.html')[0] +'_'+str(i)+'.html'\n",
    "            content = urllib2.urlopen(url_child).read().decode('gb18030') \n",
    "            soup = BeautifulSoup(content) \n",
    "            report_child = ''.join(s.text for s in soup('p'))\n",
    "            report = report + report_child\n",
    "    return year, report\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2016\n",
      "2015\n",
      "2014\n",
      "2013\n",
      "2012\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/chengjun/anaconda/lib/python2.7/site-packages/bs4/__init__.py:166: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "To get rid of this warning, change this:\n",
      "\n",
      " BeautifulSoup([your markup])\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup([your markup], \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2011\n",
      "2010\n",
      "2009\n",
      "2008\n",
      "2007\n",
      "2006\n",
      "2005\n",
      "2004\n",
      "2003\n",
      "2002\n",
      "2001\n",
      "2000\n",
      "1999\n",
      "1998\n",
      "1997\n",
      "1996\n",
      "1995\n",
      "1994\n",
      "1993\n",
      "1992\n",
      "1991\n",
      "1990\n",
      "1989\n",
      "1988\n",
      "1987\n",
      "1986\n",
      "1985\n",
      "1984\n",
      "1983\n",
      "1982\n",
      "1981\n",
      "1980\n",
      "1979\n",
      "1978\n",
      "1975\n",
      "1964\n",
      "1959\n",
      "1960\n",
      "1957\n",
      "1956\n",
      "1955\n",
      "1954\n"
     ]
    }
   ],
   "source": [
    "# 抓取47年政府工作报告内容\n",
    "reports = {}\n",
    "for link in hyperlinks:\n",
    "    year, report = crawler(link)\n",
    "    print year\n",
    "    reports[year] = report "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "url2016 = 'http://news.xinhuanet.com/fortune/2016-03/05/c_128775704.htm'\n",
    "content = urllib2.urlopen(url2016).read()\n",
    "soup = BeautifulSoup(content, 'html.parser') \n",
    "report2016 = ''.join(s.text for s in soup('p'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "with open('/Users/chengjun/github/cjc2016/data/gov_reports1954-2016.txt', 'wb') as f:\n",
    "    for r in reports:\n",
    "        line = str(r)+'\\t'+reports[r].replace('\\n', '\\t') +'\\n'\n",
    "        f.write(line.encode('utf-8'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# This is the end.\n",
    "> ## Thank you for your attention."
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}