fix issue with responses preventing spiders to be idle in engine counts

scrapy · Jul 15, 2011 · 4dadeb7 · 4dadeb7
1 parent d207c0a
commit 4dadeb7
Showing 1 changed file with 4 additions and 6 deletions.
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
@@ -179,16 +179,14 @@ def schedule(self, request, spider):
     def download(self, request, spider):
         slot = self.slots[spider]
         slot.add_request(request)
-        if isinstance(request, Response):
-            return request
         d = self._download(request, spider)
-        d.addCallback(self.download, spider)
-        d.addBoth(self._remove_request, slot, request)
+        d.addBoth(self._downloaded, slot, request, spider)
         return d
 
-    def _remove_request(self, _, slot, request):
+    def _downloaded(self, response, slot, request, spider):
         slot.remove_request(request)
-        return _
+        return self.download(response, spider) \
+                if isinstance(response, Request) else response
 
     def _download(self, request, spider):
         slot = self.slots[spider]