Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
108 lines (74 sloc) 3.25 KB
import scraperwiki
import mechanize
import lxml.html
def fetch_dom(startUrl, addPath):
scrape = scraperwiki.scrape(startUrl + addPath)
result = lxml.html.fromstring(scrape)
return result
start_url = ''
scrape = scraperwiki.scrape(start_url)
dom = lxml.html.fromstring(scrape)
#1. get the main navigation links
nav_links = []
for el in dom.cssselect('div#nav-product ul li a'):
one_link = el.attrib['href']
print nav_links
#2. go through each section (nav destination, and start collecting each item there)
ourData = []
for curr_path in nav_links:
page_dom = fetch_dom(start_url, curr_path)
#2a, get category
for el in page_dom.cssselect('div#bodycontent > div.breadcrumbs'):
category = el.text_content()
print category
#3. Go through each item and start gathering
item_links = []
for elem in page_dom.cssselect(' a'):
item_link = elem.attrib['href']
for curr_item in item_links:
item_dom = fetch_dom(start_url, curr_item)
#3a. get item ID
for element0 in item_dom.cssselect('div.code em'):
itemId = element0.text
for element1 in item_dom.cssselect('#item-contenttitle'):#might be going by multiple elem. hence errors.
itemTitle = element1.text
if item_dom.cssselect('div.price'):
for element2 in item_dom.cssselect('div.price'):
itemPrice = element2.text
for element2 in item_dom.cssselect('div.price-bold'):
itemPrice = element2.text
if item_dom.cssselect(''):
for element2b in item_dom.cssselect(' em'):
salePrice = element2b.text
salePrice = 'na'
for element3 in item_dom.cssselect('div#caption div'):
itemDescription = element3.text_content() #using .text_content() is prefereable to .text as the latter stops at <br> elements, not retrieving all text.
itemDescriptionHtml = lxml.html.tostring(element3)
for element4 in item_dom.cssselect('div#itemarea>a'):
itemImage = element4.attrib['href']
for element5 in item_dom.cssselect('div#itemarea>a>img'):
itemThumb = element5.attrib['src']
itemShipping = []
for element6 in item_dom.cssselect('div.itemform select'):
for opt in element6:
uniqueId = itemId + itemPrice + '$' + itemTitle
"category": category,
"id": itemId,
"unique_id": uniqueId,
"url": start_url+curr_item,
"title": itemTitle,
"description": itemDescription,
"descriptionHtml": itemDescriptionHtml,
"price": itemPrice,
"sale_price": salePrice,
"item_shipping": itemShipping,
"image": itemImage,
"img_thumb": itemThumb
})["unique_id"], data=ourData, table_name="igal_products")