Skip to content
Permalink
Browse files

Patches Twisted issue while closing the connection pool on HTTPDownlo…

…adHandler
  • Loading branch information
curita authored and nramirezuy committed Jan 20, 2015
1 parent eeb589a commit d0bf957069734fac0aba035ef0560e83c3a1e6c9
Showing with 19 additions and 1 deletion.
  1. +19 −1 scrapy/core/downloader/handlers/http11.py
@@ -29,14 +29,32 @@ def __init__(self, settings):
self._pool._factory.noisy = False
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
self._contextFactory = self._contextFactoryClass()
self._disconnect_timeout = 1

def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool)
return agent.download_request(request)

def close(self):
return self._pool.closeCachedConnections()
d = self._pool.closeCachedConnections()
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
#
# Twisted issue addressing this problem can be found here:
# https://twistedmatrix.com/trac/ticket/7738.
#
# closeCachedConnections doesn't handle external errbacks, so we'll
# issue a callback after `_disconnect_timeout` seconds.
delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])

def cancel_delayed_call(result):
if delayed_call.active():
delayed_call.cancel()
return result

d.addBoth(cancel_delayed_call)
return d


class TunnelError(Exception):

0 comments on commit d0bf957

Please sign in to comment.
You can’t perform that action at this time.