Merge pull request #159 from Lukas0907/error-handling

Adjustment to response caching, logging

feeds/cache.py

@@ -7,7 +7,7 @@
 logger = logging.getLogger(__name__)
 
 
-IGNORE_HTTP_CODES = [403, 404, 500, 502, 503, 504]
+IGNORE_HTTP_CODES = [403, 404] + list(range(500, 600))
 
 
 def read_meta(root):

feeds/spidermiddlewares.py

@@ -15,9 +15,9 @@ def from_crawler(cls, crawler):
 
     def process_spider_exception(self, response, exception, spider):
         if isinstance(exception, HttpError):
-            if response.status in [500, 502, 503, 504]:
-                # These status codes are usually induced by overloaded sites,
-                # updates, short downtimes, etc. and are not that relevant.
+            if response.status >= 500:
+                # Transient errors usually caused by overloaded sites, updates, short
+                # downtimes, etc.
                 lgr = logger.info
             else:
                 lgr = logger.warning

feeds/spiders/generic.py

@@ -9,6 +9,10 @@
 from feeds.loaders import FeedEntryItemLoader
 from feeds.spiders import FeedsSpider
 
+# Readability's output is not that interesting to justify log level "INFO".
+import readability.readability
+readability.readability.log.info = readability.readability.log.debug
+
 
 class GenericSpider(FeedsSpider):
     name = "generic"