Update scraper.py

paulbradshaw · Jan 25, 2018 · 662ecfc · 662ecfc
1 parent 9168cbd
commit 662ecfc
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/scraper.py b/scraper.py
@@ -18,8 +18,9 @@
   print link
   print link.text
   print link.text_content()
-  print link.attrib['href']
-  fullurl = baseurl+link.attrib['href'].split("?")[0]
+  cleanurl = link.attrib['href'].split("?")[0]
+  print cleanurl
+  fullurl = baseurl+cleanurl
   linkslist.append(fullurl)
 
 for url in linkslist: