From 62ff4f8bbd88ee16650c4f8ee49fdff8dc194eeb Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 25 Oct 2016 16:25:20 +0200 Subject: [PATCH 1/6] replace with treq --- vumi/tests/fake_connection.py | 7 ++++--- vumi/utils.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/vumi/tests/fake_connection.py b/vumi/tests/fake_connection.py index 24bfd5767..327b83fd1 100644 --- a/vumi/tests/fake_connection.py +++ b/vumi/tests/fake_connection.py @@ -271,7 +271,7 @@ def __init__(self, handler): def endpoint(self): return self.fake_server.endpoint - def get_agent(self, reactor=None, contextFactory=None): + def get_agent(self, reactor=None, pool=None, contextFactory=None): """ Returns an IAgent that makes requests to this fake server. """ @@ -297,6 +297,7 @@ def render_PUT(self, request): class ProxyAgentWithContext(ProxyAgent): - def __init__(self, endpoint, reactor=None, contextFactory=None): + def __init__(self, endpoint, reactor=None, pool=None, contextFactory=None): self.contextFactory = contextFactory # To assert on in tests. - super(ProxyAgentWithContext, self).__init__(endpoint, reactor=reactor) + super(ProxyAgentWithContext, self).__init__( + endpoint, reactor=reactor, pool=pool) diff --git a/vumi/utils.py b/vumi/utils.py index 0de4c0a25..f67c41800 100644 --- a/vumi/utils.py +++ b/vumi/utils.py @@ -19,6 +19,8 @@ from twisted.web.iweb import IBodyProducer from twisted.web.http import PotentialDataLoss from twisted.web.resource import Resource +from treq._utils import default_pool, default_reactor +from treq.client import HTTPClient from vumi.errors import VumiError @@ -119,6 +121,34 @@ def connectionLost(self, reason): def http_request_full(url, data=None, headers={}, method='POST', timeout=None, data_limit=None, context_factory=None, agent_class=None, reactor=None): + """ + This is a drop in replacement for the original `http_request_full` method + but it has its internals completely replaced by treq. Treq supports SNI + and our implementation does not for some reason. Also, we do not want + to continue maintaining this because we're favouring treq everywhere + anyway. + + """ + agent_class = agent_class or Agent + if reactor is None: + # The import replaces the local variable. + from twisted.internet import reactor + pool = default_pool(reactor, pool=None, persistent=False) + agent = agent_class(reactor, pool=pool) + client = HTTPClient(agent) + + def handle_response(response): + return SimplishReceiver(response, data_limit).deferred + + d = client.request(method, url, + headers=headers, data=data, timeout=timeout) + d.addCallback(handle_response) + return d + + +def old_http_request_full(url, data=None, headers={}, method='POST', + timeout=None, data_limit=None, context_factory=None, + agent_class=None, reactor=None): if reactor is None: # The import replaces the local variable. from twisted.internet import reactor From b527f6949b2f7a4dd54bca4d3cea5bf3662126a5 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 25 Oct 2016 16:30:47 +0200 Subject: [PATCH 2/6] use the context factory --- vumi/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vumi/utils.py b/vumi/utils.py index f67c41800..84397a692 100644 --- a/vumi/utils.py +++ b/vumi/utils.py @@ -134,7 +134,8 @@ def http_request_full(url, data=None, headers={}, method='POST', # The import replaces the local variable. from twisted.internet import reactor pool = default_pool(reactor, pool=None, persistent=False) - agent = agent_class(reactor, pool=pool) + context_factory = context_factory or WebClientContextFactory() + agent = agent_class(reactor, pool=pool, contextFactory=context_factory) client = HTTPClient(agent) def handle_response(response): From d35e97f73f64f05cab2592bf50bfa3aed5cc279d Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 25 Oct 2016 16:33:09 +0200 Subject: [PATCH 3/6] fix custom context factory tests --- vumi/tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vumi/tests/test_utils.py b/vumi/tests/test_utils.py index a70a3ccfc..7478a3e0b 100644 --- a/vumi/tests/test_utils.py +++ b/vumi/tests/test_utils.py @@ -234,9 +234,9 @@ def test_http_request_with_custom_context_factory(self): ctxt = WebClientContextFactory() - def stashing_factory(reactor, contextFactory=None): + def stashing_factory(reactor, contextFactory=None, pool=None): agent = self.fake_http.get_agent( - reactor, contextFactory=contextFactory) + reactor, contextFactory=contextFactory, pool=pool) agents.append(agent) return agent From 12fbbda0df33a59e515b8982311535ec59e82282 Mon Sep 17 00:00:00 2001 From: Simon de Haan Date: Tue, 25 Oct 2016 16:53:39 +0200 Subject: [PATCH 4/6] fix test_http_request_full_timeout_after_connect --- vumi/utils.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/vumi/utils.py b/vumi/utils.py index 84397a692..49e8e7bde 100644 --- a/vumi/utils.py +++ b/vumi/utils.py @@ -141,8 +141,29 @@ def http_request_full(url, data=None, headers={}, method='POST', def handle_response(response): return SimplishReceiver(response, data_limit).deferred - d = client.request(method, url, - headers=headers, data=data, timeout=timeout) + d = client.request(method, url, headers=headers, data=data) + + if timeout is not None: + cancelling_on_timeout = [False] + + def raise_timeout(reason): + if not cancelling_on_timeout[0] or reason.check(HttpTimeoutError): + return reason + return Failure(HttpTimeoutError("Timeout while connecting")) + + def cancel_on_timeout(): + cancelling_on_timeout[0] = True + d.cancel() + + def cancel_timeout(r, delayed_call): + if delayed_call.active(): + delayed_call.cancel() + return r + + d.addErrback(raise_timeout) + delayed_call = reactor.callLater(timeout, cancel_on_timeout) + d.addCallback(cancel_timeout, delayed_call) + d.addCallback(handle_response) return d From 82015b2d8fd6941c6f3dd9cbddf3d2c869ebbf4b Mon Sep 17 00:00:00 2001 From: Kaitlyn Crawford Date: Thu, 20 Apr 2017 14:17:52 +0200 Subject: [PATCH 5/6] Add callback to request before handling timeout --- vumi/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vumi/utils.py b/vumi/utils.py index 49e8e7bde..68a623637 100644 --- a/vumi/utils.py +++ b/vumi/utils.py @@ -142,6 +142,7 @@ def handle_response(response): return SimplishReceiver(response, data_limit).deferred d = client.request(method, url, headers=headers, data=data) + d.addCallback(handle_response) if timeout is not None: cancelling_on_timeout = [False] @@ -164,7 +165,6 @@ def cancel_timeout(r, delayed_call): delayed_call = reactor.callLater(timeout, cancel_on_timeout) d.addCallback(cancel_timeout, delayed_call) - d.addCallback(handle_response) return d From 927669d93f0c006ae9536dffe8a6b32d12f1950f Mon Sep 17 00:00:00 2001 From: Kaitlyn Crawford Date: Thu, 4 May 2017 15:28:33 +0200 Subject: [PATCH 6/6] Replace vulnerable HTTPS policy for verifying certificates --- vumi/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vumi/utils.py b/vumi/utils.py index 68a623637..9524bc71f 100644 --- a/vumi/utils.py +++ b/vumi/utils.py @@ -13,7 +13,7 @@ from twisted.internet import protocol from twisted.internet.defer import succeed from twisted.python.failure import Failure -from twisted.web.client import Agent, ResponseDone, WebClientContextFactory +from twisted.web.client import Agent, ResponseDone, BrowserLikePolicyForHTTPS from twisted.web.server import Site from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer @@ -134,7 +134,7 @@ def http_request_full(url, data=None, headers={}, method='POST', # The import replaces the local variable. from twisted.internet import reactor pool = default_pool(reactor, pool=None, persistent=False) - context_factory = context_factory or WebClientContextFactory() + context_factory = context_factory or BrowserLikePolicyForHTTPS() agent = agent_class(reactor, pool=pool, contextFactory=context_factory) client = HTTPClient(agent) @@ -176,7 +176,7 @@ def old_http_request_full(url, data=None, headers={}, method='POST', from twisted.internet import reactor if agent_class is None: agent_class = Agent - context_factory = context_factory or WebClientContextFactory() + context_factory = context_factory or BrowserLikePolicyForHTTPS() agent = agent_class(reactor, contextFactory=context_factory) d = agent.request(method, url,