@@ -2,6 +2,7 @@ import pprint import re import unittest +import csv class Deposit: def __init__(self, aTuple): @@ -19,20 +20,24 @@ re_text = re.compile('<td colspan="2">(.*)</td>') re_associated_legislation = re.compile('><td colspan="3" /><td><i>(.*)</i></td></tr>') deposits_html = open("deposits.html","r") +csv_writer = csv.writer(open('deposits.csv', 'w'), quoting=csv.QUOTE_ALL) for line in deposits_html: if line.startswith("<?xml"): e3e3e3 = line.split('<tr') del e3e3e3[0:5] - list_of_lists = [] for i in xrange(0, len(e3e3e3), 4): details = re_details.findall(e3e3e3[i])[0] + deposit_number = details[0] + deposit_legislature = details[1] + deposit_date = details[2] + deposit_source = details[3] + deposit_link = details[4] + deposit_file = details[5] text = re_text.findall(e3e3e3[i+1])[0] associated_legislation = re_associated_legislation.findall(e3e3e3[i+2])[0] - list_of_lists.append((details, text, associated_legislation)) - # pprint.pprint(list_of_lists[4]) - z = Deposit(list_of_lists[4]) - print z.download_text + csv_writer.writerow([deposit_number, deposit_legislature, deposit_date, deposit_source, deposit_link, deposit_file, text, associated_legislation]) + scraper.py bdb999096ab67ddb036643f7d6edd79bd68e7ece Robert Brook robert@robertbrook.gateway.2wire.net http://github.com/robertbrook/depositscraper/commit/8e2bd8ea32f6919998921aa19c75fcdcce353d19 8e2bd8ea32f6919998921aa19c75fcdcce353d19 2009-02-27T09:24:14-08:00 2009-02-27T09:24:14-08:00 comma sep 13368b84a4a410bf9f5ae7ad3a025f2ff2d40441 Robert Brook robert@robertbrook.gateway.2wire.net