Update scraper.py

aniri · Oct 29, 2017 · 6da8b56 · 6da8b56
1 parent 995cf43
commit 6da8b56
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/scraper.py b/scraper.py
@@ -178,9 +178,10 @@ def get_contacts(self, text):
         return contact
 
 process = CrawlerProcess({
-    'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
+    'USER_AGENT': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
     'LOG_LEVEL' : 'WARNING',
-    'DOWNLOAD_DELAY' : 0.25
+    'DOWNLOAD_DELAY' : 0.25,
+    'ROBOTSTXT_OBEY' : true
 })
 process.crawl(SanatateSpider)
 process.start()