Update scraper.py

hicham20201441 · Aug 27, 2020 · 81d8c68 · 81d8c68
1 parent 0acba9b
commit 81d8c68
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/scraper.py b/scraper.py
@@ -12,12 +12,12 @@
 root = lxml.html.fromstring(html)
 urls=[e.get("href") for e in root.cssselect("a")]
 while(len(urls)>0):
-  if url in urls[0]:
+  if url in urls[0] and urls!=url:
     ur=urls[0]
     print("scraping: "+ur)
     html1= scraperwiki.scrape(urls[0])
     root1 = lxml.html.fromstring(html1)
-    urls.pop()
+    urls.pop(0)
     newrls=[e.get("href") for e in root1.cssselect("a")]
     urls=urls+newrls
     try: