Skip to content
Permalink
Browse files

Merge pull request #1 from blablupcom/patch-1

Update scraper.py
  • Loading branch information
woodbine committed Jan 4, 2016
2 parents 7798c91 + fa57f0d commit 6e56094bdd379566a0a09061aa057adb9327a3c9
Showing with 134 additions and 26 deletions.
  1. +134 −26 scraper.py
@@ -1,41 +1,149 @@
# -*- coding: utf-8 -*-

#### IMPORTS 1.0

import os
import re
import scraperwiki
import urllib2
from datetime import datetime
from bs4 import BeautifulSoup

# Set up variables


#### FUNCTIONS 1.0

def validateFilename(filename):
filenameregex = '^[a-zA-Z0-9]+_[a-zA-Z0-9]+_[a-zA-Z0-9]+_[0-9][0-9][0-9][0-9]_[0-9QY][0-9]$'
dateregex = '[0-9][0-9][0-9][0-9]_[0-9QY][0-9]'
validName = (re.search(filenameregex, filename) != None)
found = re.search(dateregex, filename)
if not found:
return False
date = found.group(0)
now = datetime.now()
year, month = date[:4], date[5:7]
validYear = (2000 <= int(year) <= now.year)
if 'Q' in date:
validMonth = (month in ['Q0', 'Q1', 'Q2', 'Q3', 'Q4'])
elif 'Y' in date:
validMonth = (month in ['Y1'])
else:
try:
validMonth = datetime.strptime(date, "%Y_%m") < now
except:
return False
if all([validName, validYear, validMonth]):
return True


def validateURL(url):
try:
r = urllib2.urlopen(url)
count = 1
while r.getcode() == 500 and count < 4:
print ("Attempt {0} - Status code: {1}. Retrying.".format(count, r.status_code))
count += 1
r = urllib2.urlopen(url)
sourceFilename = r.headers.get('Content-Disposition')

if sourceFilename:
ext = os.path.splitext(sourceFilename)[1].replace('"', '').replace(';', '').replace(' ', '')
else:
ext = os.path.splitext(url)[1]
validURL = r.getcode() == 200
validFiletype = ext.lower() in ['.csv', '.xls', '.xlsx']
return validURL, validFiletype
except:
print ("Error validating URL.")
return False, False


def validate(filename, file_url):
validFilename = validateFilename(filename)
validURL, validFiletype = validateURL(file_url)
if not validFilename:
print filename, "*Error: Invalid filename*"
print file_url
return False
if not validURL:
print filename, "*Error: Invalid URL*"
print file_url
return False
if not validFiletype:
print filename, "*Error: Invalid filetype*"
print file_url
return False
return True


def convert_mth_strings ( mth_string ):
month_numbers = {'JAN': '01', 'FEB': '02', 'MAR':'03', 'APR':'04', 'MAY':'05', 'JUN':'06', 'JUL':'07', 'AUG':'08', 'SEP':'09','OCT':'10','NOV':'11','DEC':'12' }
for k, v in month_numbers.items():
mth_string = mth_string.replace(k, v)
return mth_string


#### VARIABLES 1.0

entity_id = "E5044_MBC_gov"
url = "http://www.merton.gov.uk/council/dp-foi/opendata/spending.htm"
archive_url = 'http://www.merton.gov.uk/council/dp-foi/opendata/older-spend.htm'
errors = 0
data = []


#### READ HTML 1.0

# Set up functions
def convert_mth_strings ( mth_string ):
month_numbers = {'JAN': '01', 'FEB': '02', 'MAR':'03', 'APR':'04', 'MAY':'05', 'JUN':'06', 'JUL':'07', 'AUG':'08', 'SEP':'09','OCT':'10','NOV':'11','DEC':'12' }
#loop through the months in our dictionary
for k, v in month_numbers.items():
#then replace the word with the number
mth_string = mth_string.replace(k, v)
return mth_string

# pull down the content from the webpage
html = urllib2.urlopen(url)
soup = BeautifulSoup(html)
soup = BeautifulSoup(html, 'lxml')

# find all entries with the required class
table = soup.find('table',{'class':'t1'}) # find docs table
#### SCRAPE DATA

table = soup.find('table',{'class':'t1'})
links = table.findAll('a', href=True)

for link in links:
url = "http://www.merton.gov.uk" + link['href']
if '.csv' in url:
title = link.contents[0]
# create the right strings for the new filename
csvYr = title.split(' ')[1]
csvMth = title.split(' ')[0][:3]
csvMth = csvMth.upper()
csvMth = convert_mth_strings(csvMth);
filename = entity_id + "_" + csvYr + "_" + csvMth + ".csv"
todays_date = str(datetime.now())
scraperwiki.sqlite.save(unique_keys=['l'], data={"l": url, "f": filename, "d": todays_date })
print filename
url = "http://www.merton.gov.uk" + link['href']
if '.csv' in url:
title = link.contents[0]
csvYr = title.split(' ')[1]
csvMth = title.split(' ')[0][:3]
csvMth = convert_mth_strings(csvMth.upper())
todays_date = str(datetime.now())
data.append([csvYr, csvMth, url])
archive_html = urllib2.urlopen(archive_url)
archive_soup = BeautifulSoup(archive_html, 'lxml')
table = archive_soup.find('table',{'class':'t1'})
links = table.findAll('a', href=True)
for link in links:
url = "http://www.merton.gov.uk" + link['href']
if '.csv' in url:
title = link.contents[0]
csvYr = title.split(' ')[1]
csvMth = title.split(' ')[0][:3]
csvMth = convert_mth_strings(csvMth.upper())
todays_date = str(datetime.now())
data.append([csvYr, csvMth, url])

#### STORE DATA 1.0

for row in data:
csvYr, csvMth, url = row
filename = entity_id + "_" + csvYr + "_" + csvMth
todays_date = str(datetime.now())
file_url = url.strip()

valid = validate(filename, file_url)

if valid == True:
scraperwiki.sqlite.save(unique_keys=['l'], data={"l": file_url, "f": filename, "d": todays_date })
print filename
else:
errors += 1

if errors > 0:
raise Exception("%d errors occurred during scrape." % errors)


#### EOF

0 comments on commit 6e56094

Please sign in to comment.
You can’t perform that action at this time.