Create scraper.py

walinchus · Jun 14, 2017 · 066762e · 066762e
1 parent 4c99d1f
commit 066762e
Showing 1 changed file with 8 additions and 7 deletions.
diff --git a/scraper.py b/scraper.py
@@ -110,10 +110,7 @@ def scrape_table(root):
 
 
 def scrape_and_look_for_next_link(url):
-    html = scraperwiki.scrape(url)
-        #print html
-    root = lxml.html.fromstring(html)
-    scrape_table(root)
+
         #for next_link in range (1, 744):
             #print next_link
         #Add_Case_No()
@@ -122,9 +119,13 @@ def scrape_and_look_for_next_link(url):
     for i in range(1,744):
         next_link +=1
         print next_link
-    next_url = base_url+'GetCaseInformation.aspx?db=garfield&number=CF-2011-'+str(next_link)
-    print next_url
-    scrape_and_look_for_next_link(next_url)
+        html = scraperwiki.scrape(url)
+        #print html
+        root = lxml.html.fromstring(html)
+        scrape_table(root)
+        next_url = base_url+'GetCaseInformation.aspx?db=garfield&number=CF-2011-'+str(next_link)
+        print next_url
+        scrape_and_look_for_next_link(next_url)
 
 # ---------------------------------------------------------------------------
 # START HERE: define your starting URL - then