Skip to content
Permalink
Browse files

Save page progress more frequently

  • Loading branch information...
richardingham committed May 10, 2014
1 parent fd2366d commit bf290795b0919ef380a628adce76da028aa01265
Showing with 2 additions and 1 deletion.
  1. +2 −1 scraper.py
@@ -41,6 +41,8 @@ def shallow_scrape():
### get new page
try:
resultspage = br.follow_link(text="Next")
sqlite.save_var('last_page', c)

c += 1
if c % 2 == 0:
time.sleep(10)
@@ -49,7 +51,6 @@ def shallow_scrape():
c = 1
break

sqlite.save_var('last_page', c - 1)

keys_to_keep = [
'Local Authority', 'Type of Establishment', 'Locality', 'Establishment Number', 'School Capacity', 'Statutory Lowest Pupil Age',

0 comments on commit bf29079

Please sign in to comment.
You can’t perform that action at this time.