Skip to content

Commit

Permalink
Update scraper.py
Browse files Browse the repository at this point in the history
  • Loading branch information
bethanjhn committed Jul 30, 2019
1 parent c711870 commit 3d1852d
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ def scrape_table(jim):

# # scrape_and_look_for_next_link function: calls the scrape_table
# # function, then hunts for a 'next' link: if one is found, calls itself again
def scrape_and_look_for_next_link(url):
html = scraperwiki.scrape(url)
def scrape_and_look_for_next_link(starting_url):
html = scraperwiki.scrape(starting_url)
print html
jim = lxml.html.fromstring(html)
scrape_table(jim)
root = lxml.html.fromstring(html)
scrape_table(root)
# Below would find a next button on the page and select it, then loop through that page etc
# next_link = root.cssselect("a.next")
# print next_link
Expand Down

0 comments on commit 3d1852d

Please sign in to comment.