/
scraper.py
72 lines (53 loc) · 1.99 KB
/
scraper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
##########################
# zygo
##########################
import scraperwiki
import lxml.html
import httplib2
from BeautifulSoup import BeautifulSoup, SoupStrainer
import re
import itertools
skipsearches = 0 #1 to skip previous searches
skiprecords = 0 #1 to skip previous result records
maxcountys = 10 #max countys to search for with every cycle. currently 10
baseurl = 'http://www.freereg.org.uk/cgi/'
surnames = ['Muchall']
forename = 'null'
def mygrouper(n, iterable):
args = [iter(iterable)] * n
return list(([e for e in t if e != None] for t in itertools.izip_longest(*args)))
page = scraperwiki.scrape(baseurl + "Search.pl")
root = lxml.html.fromstring(page)
radios = root.cssselect("input[name='RecordType']")
recordtypes = [y.get('value') for y in radios]
options = root.cssselect("select[name='County'] option")
allcountys = [y.get('value') for y in options]
countys = mygrouper(maxcountys,allcountys)
data = {}
myerrors = 0
myskips = 0
myadded = 0
allsurnames = "".join(str(x) for x in surnames)
mysql = "SurnameRecordtypeCounty from Searches"
try:
presearches = scraperwiki.sqlite.select(mysql)
except:
presearches = []
for recordtype in recordtypes:
for surname in surnames:
mysql = "Url from " + forename + allsurnames
try:
predata = scraperwiki.sqlite.select(mysql)
except:
predata = []
for county in countys:
searchurl = baseurl + "Search.pl?RecordType=" + recordtype + "&Surname=" + surname + "&Action=Search&County=" + "&County=".join(str(x) for x in county)
countyonlystr = "".join(str(x) for x in county)
searchdata = {'SurnameRecordtypeCounty': (surname + recordtype + countyonlystr)}
if searchdata in presearches and skipsearches:
myskips = myskips + 1
print(myskips, " Skipped ", surname, recordtype, county)
continue
print "Error Count = ", myerrors
print "Skipped Count = ", myskips
print "Added Count = ", myadded