@@ -1,6 +1,7 @@ #!/usr/bin/python import pprint import re +import unittest class Deposit: def __init__(self, aTuple): @@ -13,10 +14,9 @@ class Deposit: self.download_text = aTuple[0][5] self.associated_legislation = aTuple[2] - -valign = re.compile(' valign="top"><td bgcolor="#e3e3e3">(.*)</td><td bgcolor="#e3e3e3">(.*)</td><td bgcolor="#e3e3e3">(.*)</td><td colspan="1" bgcolor="#e3e3e3" width="45%">(.*)</td><td bgcolor="#e3e3e3" width="25%" align="right"><A HREF="(.*)">(.*)</A></td></tr>') -td_colspan = re.compile('<td colspan="2">(.*)</td>') -td_i = re.compile('><td colspan="3" /><td>(.*)</td></tr>') +re_details = re.compile(' valign="top"><td bgcolor="#e3e3e3">(.*)</td><td bgcolor="#e3e3e3">(.*)</td><td bgcolor="#e3e3e3">(.*)</td><td colspan="1" bgcolor="#e3e3e3" width="45%">(.*)</td><td bgcolor="#e3e3e3" width="25%" align="right"><A HREF="(.*)">(.*)</A></td></tr>') +re_text = re.compile('<td colspan="2">(.*)</td>') +re_associated_legislation = re.compile('><td colspan="3" /><td>(.*)</td></tr>') deposits_html = open("deposits.html","r") @@ -26,14 +26,13 @@ for line in deposits_html: del e3e3e3[0:5] list_of_lists = [] for i in xrange(0, len(e3e3e3), 4): - first = valign.findall(e3e3e3[i])[0] - second = td_colspan.findall(e3e3e3[i+1])[0] - third = td_i.findall(e3e3e3[i+2])[0] - list_of_lists.append((first, second, third)) - d = Deposit((first, second, third)) - print d.legislature + details = re_details.findall(e3e3e3[i])[0] + text = re_text.findall(e3e3e3[i+1])[0] + associated_legislation = re_associated_legislation.findall(e3e3e3[i+2])[0] + list_of_lists.append((details, text, associated_legislation)) # pprint.pprint(list_of_lists[4]) z = Deposit(list_of_lists[4]) - - print z.text + print z.download_text + + scraper.py c102388b26899bcfefa5578d4a33286a30363d0c Robert Brook mail@robertbrook.com http://github.com/robertbrook/depositscraper/commit/bdb999096ab67ddb036643f7d6edd79bd68e7ece bdb999096ab67ddb036643f7d6edd79bd68e7ece

2009-01-02T09:40:07-08:00

trying to get unittest working 4b121615c499583892fe8d4474da367e5ca32acd Robert Brook mail@robertbrook.com