Create scraper.py

scraper.py

@@ -93,20 +93,21 @@ def scrape_table(root):
 61 #pass that new concatenated URL to a function, 'scrape_page', which is scripted above
 62 scrape_page(next_link)'''
 
-'''def Add_Case_No(next_link):
+def Add_Case_No(next_link):
+    for next_link in range (0,744):
     return next_link + 1
     print "the next case number is:", next_link
-    next_link = 'GetCaseInformation.aspx?db=garfield&number=CF-2011-' + str(next_link)'''
+    next_link = 'GetCaseInformation.aspx?db=garfield&number=CF-2011-' + str(next_link)
 
 def scrape_and_look_for_next_link(url):
     html = scraperwiki.scrape(url)
     #print html
     root = lxml.html.fromstring(html)
     scrape_table(root)
-    for next_link in range (1, 744):
-        print next_link
+    #for next_link in range (1, 744):
+        #print next_link
         if next_link:
-            next_url = base_url+'GetCaseInformation.aspx?db=garfield&number=CF-2011-'+str(next_link)
+            next_url = base_url+str(next_link)
             print next_url
             scrape_and_look_for_next_link(next_url)
 
@@ -117,7 +118,7 @@ def scrape_and_look_for_next_link(url):
 base_url = 'http://www.oscn.net/dockets/'
 starting_url = urlparse.urljoin(base_url, 'GetCaseInformation.aspx?db=garfield&number=CF-2011-1')
 print starting_url
-#Add_Case_No(next_link)
+Add_Case_No(next_link)
 scrape_and_look_for_next_link(starting_url)