Skip to content
Browse files

Patches Twisted issue while closing the connection pool on HTTPDownlo…

  • Loading branch information
curita authored and nramirezuy committed Jan 20, 2015
1 parent eeb589a commit d0bf957069734fac0aba035ef0560e83c3a1e6c9
Showing with 19 additions and 1 deletion.
  1. +19 −1 scrapy/core/downloader/handlers/
@@ -29,14 +29,32 @@ def __init__(self, settings):
self._pool._factory.noisy = False
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
self._contextFactory = self._contextFactoryClass()
self._disconnect_timeout = 1

def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool)
return agent.download_request(request)

def close(self):
return self._pool.closeCachedConnections()
d = self._pool.closeCachedConnections()
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
# Twisted issue addressing this problem can be found here:
# closeCachedConnections doesn't handle external errbacks, so we'll
# issue a callback after `_disconnect_timeout` seconds.
delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])

def cancel_delayed_call(result):
return result

return d

class TunnelError(Exception):

0 comments on commit d0bf957

Please sign in to comment.
You can’t perform that action at this time.