Update http11.py

scrapy#5618
PsykotropyK · Sep 6, 2022 · 12af263 · 12af263
1 parent 0da7a79
commit 12af263
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
@@ -38,8 +38,9 @@ def __init__(self, settings, crawler=None):
         self._crawler = crawler
 
         from twisted.internet import reactor
-        self._pool = HTTPConnectionPool(reactor, persistent=True)
+        self._pool = HTTPConnectionPool(reactor, persistent=settings.getbool('DOWNLOAD_CONNECTION_IS_PERSISTENT'))
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
+        self._pool.cachedConnectionTimeout = settings.getint('DOWNLOAD_IDLE_CONNECTION_TIMEOUT')
         self._pool._factory.noisy = False
 
         self._contextFactory = load_context_factory_from_settings(settings, crawler)
@@ -54,6 +55,9 @@ def from_crawler(cls, crawler):
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
+        if ('reset_connections', True) in request.meta:
+            self._pool.closeCachedConnections()
+
         agent = ScrapyAgent(
             contextFactory=self._contextFactory,
             pool=self._pool,