Update scraper.py

aniri · Oct 30, 2017 · b18ffc0 · b18ffc0
1 parent 1a26253
commit b18ffc0
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/scraper.py b/scraper.py
@@ -181,7 +181,8 @@ def get_contacts(self, text):
     'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
     'LOG_LEVEL' : 'WARNING',
     'DOWNLOAD_DELAY' : 2.0,
-    'ROBOTSTXT_OBEY' : True
+    'ROBOTSTXT_OBEY' : True,
+    'CONCURRENT_REQUESTS' : 2
 })
 process.crawl(SanatateSpider)
 process.start()