Permalink
Browse files

Shift data saving to tightest loop for debug purposes

  • Loading branch information...
Duncan Parkes
Duncan Parkes committed May 14, 2015
1 parent 6e8c4a0 commit 566b657f0020ff4bbe6285d813d996c75ced83da
Showing with 3 additions and 2 deletions.
  1. +3 −2 scraper.py
View
@@ -14,7 +14,6 @@
for term_name, term_url in terms:
data = []
while term_url:
print term_url
term_resp = requests.get(term_url)
@@ -23,6 +22,7 @@
trs = term_root.cssselect('.jsn-infotable')[0].cssselect('tr')[1:]
for tr in trs:
data = []
member = {}
member['term'] = term_name
member['chamber'] = 'National Assembly'
@@ -49,7 +49,8 @@
data.append(member)
scraperwiki.sqlite.save(unique_keys=['id'], data=data)
next_links = term_root.cssselect('a[title=Next]')
term_url = urljoin(term_url, next_links[0].get('href')) if next_links else None
scraperwiki.sqlite.save(unique_keys=['id'], data=data)

0 comments on commit 566b657

Please sign in to comment.