/
scraper.py
224 lines (159 loc) · 6.54 KB
/
scraper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
#SQLite quirks, system takes query and wraps it in such a way that special things cannot happen
#scraperwiki does not allow indexes or require setting up cursors or committing
#also automatically opens your database
#Important Links,
#http://www.hobbyking.com/hobbyking/store/uh_viewItem.asp?idProduct=$productId
#Hobyking link mapping,
#Categories, __488__484__Cars_Parts-1_10th_Scale.html -> pc=484&idCategory=488
#Use dictionaries for core storage because you can store initial values and then
#when updating records you can update the dictionary and the commit it
#Dicts can update Dicts...
#Converting dicts to lists the dict gets sorted
#Create unique index does not work
import scraperwiki as scraper
import scraperwiki.sqlite as lite
import lxml.html as parser
from datetime import datetime
from urlparse import parse_qsl, urlparse
from datetime import date
import re
import Queue
import threading
queue = Queue.Queue()
######################### GLOBAL PARAMETERS #################################
scrapesite = "http://secure.social.yoox.it/api/Items.API/ADIDASBY_US/ItemsWithAvailability/get.json"
scrapeprod = "http://secure.social.yoox.it/api/Items.API/ADIDASBY_US/ItemsWithAvailability/get.json?cod8="
catpagesrx = "curPage"
prodpagesrx = "(?<![0-9])__([0-9]*)__(?![0-9])"
scrapecat = [ #cod8, ukipcategory, ukcurpage, lstdate
('86', 1, "date")
]
CoreKeys = [
['cod8' , 'INT' ], #Product ID
['Value' , 'REAL'], #Colorway and product code
]
PageParser = [
['ukLink' , 'TEXT'], #Product links yet to be explored
['LstDate' , 'TEXT'] #Last date parsed, parse once per month
]
CategoryParser = [
['ukidCategory', 'INT' ], #url parameter
['ukcurPage' , 'INT' ], #Page that has been parsed
['LstDate' , 'TEXT'] #Last date parsed, parse once per day
]
######################## CLASSES ################################
class core_store(object):
def __init__(self,table,keys):
self._table = table
self._keys = keys
#self.core_reset()
self.setup()
def setup(self):
if len(lite.table_info(self._table)) > 0: return 0
query = "CREATE TABLE IF NOT EXISTS "+self._table+" ("
for key in self._keys:
query = query+" "+key[0]+" "+key[1]
unique = ""
for key in self._keys:
if "uk" in key[0]:
unique = unique+" "+key[0]+","
print "Unique Keys in",self._table+":",unique
query = query+" UNIQUE("+unique+") ON CONFLICT IGNORE)"
query = query.replace("INT ","INT, ")
query = query.replace("REAL ","REAL, ")
query = query.replace("TEXT ","TEXT, ")
query = query.replace(",)", ")")
print query
lite.execute(query)
def core_reset(self):
lite.execute("DROP TABLE if exists "+self._table)
def additem(self,data):
lite.execute("INSERT into "+self._table+" values (?"+(len(self._keys)-1)*", ?"+")", data+('0',)*(len(self._keys)-len(data)))
def updateitem(self,data):
print "Updating:",data
lite.execute("INSERT or REPLACE into "+self._table+" values (?"+(len(self._keys)-1)*", ?"+")", data+('0',)*(len(self._keys)-len(data)))
def getdata(self):
return lite.execute("select * from "+self._table)
def commit(self):
lite.commit()
def __close__(self):
lite.commit()
class ThreadUrl(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
host = self.queue.get()
url = urllib2.urlopen(host)
self.queue.task_done()
######################## FUNCTIONS ################################
def urltodict(url): #convert url parameters to a dict
keylist = list()
valuelist = list()
urldict = dict()
for item in parse_qsl(urlparse(url).query):
keylist.append(item[0])
valuelist.append(item[1])
urldict.update(zip(keylist,valuelist))
return urldict
def dicttourl(url,urldict): #Convert a query in a dict to a useable url
query = ""
del urldict['LstDate']
for key in urldict:
query = query+"&"+key+"="+str(urldict[key])
out = url+"?"+query
out = out.replace("?&","?")
return out
def querytodictlist(data):
out = list()
for value in data['data']:
newval = dict()
for key in data['keys']:
newval[key] = value[data['keys'].index(key)]
out.append(newval)
return out
#http://hobbyking.com/hobbyking/store/uh_listCategoriesAndProducts.asp?whl=XX&pc=&idCategory=86&curPage=2&v=&sortlist=&sortMotor=&LiPoConfig=&CatSortOrder=desc
def ParseCat(link):
CatData = urltodict(link)
CatStore.additem((CatData['idCategory'], CatData['curPage'], 0))
def ParseLink(link):
LinkStore.additem((link, 0))
def ParsePage(page):
print "Parseing Page"
for link in page.iterlinks():
url = link[2]
if type(CatRegex.search(url)).__name__ == 'SRE_Match': ParseCat(url)
elif type(PageRegex.search(url)).__name__ == 'SRE_Match': ParseLink(url)
######################## CODE ################################
#1) Load scrapecat into db
#2) Load category page
#3) Look at the bottom of a page for links to more pages in category
#4) Store a list of the categories and the pages contained
#5) Page parser reads the list of known categories
#6) Page parser gets the product links from each category page
#7) Page parser gets desired information from product page
#8) Loops till complete
#a) Setup Environment
ProductStore = core_store('ProductLinks', CoreKeys)
LinkStore = core_store('ScrapedLinks', PageParser)
CatStore = core_store('CatPages', CategoryParser)
CatRegex = re.compile(catpagesrx)
PageRegex = re.compile(prodpagesrx)
#1) Load scrapecat into db
for item in scrapecat:
CatStore.additem(item)
#2) Load category pages
CategoryPages = querytodictlist(CatStore.getdata())
for page in CategoryPages:
if not page['LstDate']==str(date.today()):
url = dicttourl(scrapesite,page).replace('uk','')
print "URL:",url
html = scraper.scrape(url)
rawpage = parser.fromstring(html)
ParsePage(rawpage)
LinkStore.commit()
CatStore.updateitem((page['ukidCategory'], page['ukcurPage'], str(date.today())))
ProductStore.__close__()
LinkStore.__close__()
CatStore.__close__()