01_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a scraper built to run on an EC2 instance to assist in TESU's open source materials accessibility initiative. It is the first of 3 notebooks used to scrape, check, create static sites, and update.\n",
    "As the name indicates, this notebook contains the scraping functionality:\n",
    "1. Retrieve .docx files stored in an S3 bucket\n",
    "2. Extract XML from those .doc files\n",
    "3. Extract all URLs contained in the document XML files\n",
    "    - Assigns a number to each file, and each link in a file.\n",
    "    - i.e.: link_001_005 is the fifth link in the first document\n",
    "4. Using requests library, request each of those URLs and store html.\n",
    "5. Stores pdfs in our S3 bucket folder \"pdfs\"\n",
    "6. Stores html content in our S3 bucket \"html_content\"\n",
    "5. Creates a Data Frame with the following columns:\n",
    "    - idx: (i.e.: \"link_001_005\")\n",
    "    - docname: Name of the document\n",
    "    - url: the url\n",
    "    - comment: contains \"youtube video\" if youtube video, \"pdf\" if pdf."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import neccessary packages. If any errors occur because of missing libraries, be sure to install them on \n",
    "# your EC2 instance.\n",
    "import os\n",
    "import requests\n",
    "from xml.etree.cElementTree import XML\n",
    "import zipfile\n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "#to store items on an AWS instance\n",
    "import boto\n",
    "import botocore\n",
    "import boto3\n",
    "from boto.s3.connection import S3Connection\n",
    "from uuid import uuid4 as uuid\n",
    "from time import sleep\n",
    "import html\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Create list of files to download to EC2 Instance\n",
    "from boto3.session import Session\n",
    "\n",
    "ACCESS_KEY='your_access_key'\n",
    "SECRET_KEY='your_secret_key'\n",
    "\n",
    "session = Session(aws_access_key_id=ACCESS_KEY,\n",
    "                  aws_secret_access_key=SECRET_KEY)\n",
    "s3 = session.resource('s3')\n",
    "your_bucket = s3.Bucket('name_of_s3_bucket')\n",
    "\n",
    "file_list = []\n",
    "for s3_file in your_bucket.objects.all():\n",
    "    #if in docs folder, and a file name greater than 5 characters, add to list.\n",
    "    if str(s3_file.key)[0:4] == \"docs\" and len(str(s3_file.key)) > 5: \n",
    "        file_list.append(str(s3_file.key))\n",
    "    else:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Save our s3 word docs to the EC2 instance\n",
    "\n",
    "bucket_name = 'name_of_s3_bucket' # replace with your bucket name\n",
    "session = Session(aws_access_key_id=ACCESS_KEY,\n",
    "                  aws_secret_access_key=SECRET_KEY)\n",
    "s3 = session.resource('s3')\n",
    "\n",
    "#for loop to loop through all files in the file_list and downlaod them to the EC2 instance.\n",
    "for word_doc in file_list:\n",
    "#word_doc is the name of our file, as a string.\n",
    "    KEY = word_doc\n",
    "    ec2_file_name = \"../ec2docs/\"+word_doc[5:]\n",
    "    \n",
    "    try:\n",
    "        s3.Bucket(bucket_name).download_file(KEY, ec2_file_name)\n",
    "    except botocore.exceptions.ClientError as e:\n",
    "        if e.response['Error']['Code'] == \"404\":\n",
    "            continue\n",
    "            #print(\"The object does not exist.\")\n",
    "        else:\n",
    "            raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#scrape!\n",
    "from boto3.session import Session\n",
    "\n",
    "bucket_name = 'name_of_s3_bucket' # replace with your bucket name\n",
    "\n",
    "session = Session(aws_access_key_id=ACCESS_KEY,\n",
    "                  aws_secret_access_key=SECRET_KEY)\n",
    "s3 = session.resource('s3')\n",
    "your_bucket = s3.Bucket(bucket_name)\n",
    "\n",
    "name_of_run = \"run2\" #### this will name the dataframe you create, can be adjusted accordingly\n",
    "file_list = os.listdir('/home/ec2-user/ec2docs')\n",
    "all_records = [] #declares all_records as a list\n",
    "# this will contain columns 'index', 'docname', 'url' 'comment'\n",
    "\n",
    "for file_count, file in enumerate(sorted(file_list)): #enumerate returns your list with each item numbered. \n",
    "                                              #so we can declare 2 variables to iterate over here, file_count and file \n",
    "                                              #sorted puts the file_list in alphanumeric order\n",
    "    #open doc, from folder 'docs', extract XML coding\n",
    "    pathway = '/home/ec2-user/ec2docs/'+file\n",
    "    document = zipfile.ZipFile(pathway)\n",
    "    xml_content = document.read('word/document.xml')\n",
    "    document.close()\n",
    "    xml_str = str(xml_content)\n",
    "    \n",
    "    #create linklist for doc, by going through the XML and finding the links\n",
    "    link_list = re.findall('>http.*?\\<',xml_str) #it returns text starting with '>http', ending with '<', inclusive.\n",
    "            \n",
    "    link_list = [x[1:-1] for x in link_list] #shaves off the last character of each item in the list. (it's a '<')\n",
    "    #replace &amp; with &, and other html entities.\n",
    "    link_list = [html.unescape(x) for x in link_list]\n",
    "    \n",
    "    for link_count, link in enumerate(link_list):\n",
    "        #request web page content\n",
    "        idx = \"link_%03d_%03d\" % ((file_count+1), (link_count+1)) #this just creates the string. ie: \"doc_001_001\"\n",
    "        docname = file\n",
    "        url = link\n",
    "        comment = \"\"\n",
    "        \n",
    "        try:\n",
    "            r = requests.get(link, headers={\"User-agent\": str(uuid())})\n",
    "            content = r.text\n",
    "            \n",
    "            if \"youtube\" in link:\n",
    "                comment = \"youtube video\"\n",
    "            elif \".pdf\" in link:\n",
    "                content = r.content\n",
    "                key = \"pdfs/\" + index + \".pdf\"\n",
    "                s3.Bucket(bucket_name).put_object(Key=key, Body=content, ContentType='application/pdf', ContentDisposition='inline')\n",
    "                comment = \"pdf file\"\n",
    "            else:\n",
    "                key = \"html_content/\" + index + \".html\"\n",
    "                s3.Bucket(bucket_name).put_object(Key=key, Body=content, ContentType='text/html')\n",
    "            \n",
    "        except:\n",
    "            comment = \"scraping error\"\n",
    "            \n",
    "        #add to df \n",
    "        #appends a dictionary with keys \"idx\", \"docname\" and \"url\" to the list all_records.\n",
    "        all_records.append({'idx': index, 'docname': file, 'url':link, 'comment':comment})\n",
    "    sleep(random.randrange(3, 9))\n",
    "#make pandas df and store in runs folder\n",
    "df_long = pd.DataFrame(all_records, columns=['idx', 'docname', 'url', 'comment'])\n",
    "csv_name = name_of_run+'.csv'\n",
    "df_long[['idx', 'docname','url','comment']].to_csv(csv_name)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}