Skip to content

Commit

Permalink
Update scraper.py
Browse files Browse the repository at this point in the history
  • Loading branch information
alexconx committed Jun 28, 2018
1 parent 1d5bbc1 commit 03938d2
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions scraper.py
Expand Up @@ -8,12 +8,13 @@
import scraperwiki

list_url = []
list_location = []

page = requests.get("https://www.gumtree.com.au/s-construction/c18346?ad=offering&ad=offering")
soup = BeautifulSoup(page.content, 'html.parser')

for a in soup.find_all('a', href=True, class_='user-ad-row user-ad-row--no-image link link--base-color-inherit link--hover-color-none link--no-underline'):
list_url.append("https://www.gumtree.com.au"+a['href'])
list_url.append("https://www.gumtree.com.au"+a['user-ad-row__location-area'])

for a in soup.find_all('a', href=True, class_='user-ad-row user-ad-row--featured-or-premium user-ad-row--no-image link link--base-color-inherit link--hover-color-none link--no-underline'):
list_url.append("https://www.gumtree.com.au"+a['href'])
Expand All @@ -32,7 +33,7 @@
for a in soup.find_all('a', href=True, class_='user-ad-row user-ad-row--featured-or-premium user-ad-row--no-image link link--base-color-inherit link--hover-color-none link--no-underline'):
list_url.append("https://www.gumtree.com.au"+a['href'])

print (list_url)
print (list_url[0])

for i in range (0, len(list_url)-1) :
scraperwiki.sqlite.save(unique_keys=['id'], data={"id" : str(i), "link": list_url[i]})

0 comments on commit 03938d2

Please sign in to comment.