Permalink
Browse files

disable concurrent requests, retry 10 times and don't cache retried p…

…ages
  • Loading branch information...
git2samus committed Feb 27, 2012
1 parent 61959b5 commit 5435d5e29e81c454a25cb20c7f81f1a7ea63c777
Showing with 6 additions and 0 deletions.
  1. +6 −0 scraping/scraping/settings.py
@@ -13,4 +13,10 @@
NEWSPIDER_MODULE = 'scraping.spiders'
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
+RETRY_HTTP_CODES = [500, 503, 504, 400, 408]
+RETRY_TIMES = 9
+
HTTPCACHE_ENABLED = True
+HTTPCACHE_IGNORE_HTTP_CODES = RETRY_HTTP_CODES
+
+CONCURRENT_REQUESTS_PER_DOMAIN = 1

0 comments on commit 5435d5e

Please sign in to comment.