Merge pull request #379 from itamarst/377.custom-web-context

Add the ability to override HTTPS TLS context

docs/releases.rst

@@ -17,6 +17,9 @@ See also :ref:`api_stability`.
 
 `git main <https://github.com/meejah/txtorcon>`_ *will likely become v23.1.0*
 
+* ``twisted.web.client.Agent`` instances now use the same HTTPS policy by default as ``twisted.web.client.Agent``.
+  It is possible to override this policy with the ``tls_context_factory`` argument, the equivalent to ``Agent``'s ``contextFactory=``.
+
 
 v23.0.0
 -------

test/test_web.py

@@ -1,6 +1,7 @@
 
 from mock import Mock
 
+from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.trial import unittest
 from twisted.internet import defer
 
@@ -14,6 +15,11 @@
 from txtorcon.circuit import TorCircuitEndpoint
 
 
+class CustomTLSContextFactory(BrowserLikePolicyForHTTPS):
+    def creatorForNetloc(self, hostname, port):
+        return super().creatorForNetloc(b"custom.domain", port)
+
+
 class WebAgentTests(unittest.TestCase):
     if not _HAVE_WEB:
         skip = "Missing web"
@@ -56,7 +62,12 @@ def test_socks_agent_tcp_host_port(self):
 
         def getConnection(key, endpoint):
             self.assertTrue(isinstance(endpoint, TorSocksEndpoint))
-            self.assertTrue(endpoint._tls)
+            self.assertIsInstance(
+                endpoint._tls,
+                BrowserLikePolicyForHTTPS().creatorForNetloc(b"host", 443).__class__
+            )
+            # This uses a Twisted private interface...
+            self.assertEqual(endpoint._tls._hostname, "meejah.ca")
             self.assertEqual(endpoint._host, u'meejah.ca')
             self.assertEqual(endpoint._port, 443)
             return defer.succeed(proto)
@@ -70,6 +81,38 @@ def getConnection(key, endpoint):
         res = yield agent.request(b'GET', b'https://meejah.ca')
         self.assertIs(res, gold)
 
+    @defer.inlineCallbacks
+    def test_socks_agent_custom_tls_context_factory(self):
+        reactor = Mock()
+        config = Mock()
+        config.SocksPort = []
+        proto = Mock()
+        gold = object()
+        proto.request = Mock(return_value=defer.succeed(gold))
+
+        def getConnection(key, endpoint):
+
+            self.assertIsInstance(
+                endpoint._tls,
+                BrowserLikePolicyForHTTPS().creatorForNetloc(b"host", 443).__class__
+            )
+            # This uses a Twisted private interface...
+            self.assertEqual(endpoint._tls._hostname, "custom.domain")
+            self.assertEqual(endpoint._host, 'meejah.ca')
+            return defer.succeed(proto)
+        pool = Mock()
+        pool.getConnection = getConnection
+
+        # do the test
+        agent = yield agent_for_socks_port(
+            reactor, config, '127.0.0.50:1234', pool=pool,
+            tls_context_factory=CustomTLSContextFactory()
+        )
+
+        # apart from the getConnection asserts...
+        res = yield agent.request(b'GET', b'https://meejah.ca')
+        self.assertIs(res, gold)
+
     @defer.inlineCallbacks
     def test_agent(self):
         reactor = Mock()
@@ -95,7 +138,12 @@ def test_agent_with_circuit(self):
         def getConnection(key, endpoint):
             self.assertTrue(isinstance(endpoint, TorCircuitEndpoint))
             target = endpoint._target_endpoint
-            self.assertTrue(target._tls)
+            self.assertIsInstance(
+                target._tls,
+                BrowserLikePolicyForHTTPS().creatorForNetloc(b"host", 443).__class__
+            )
+            # This uses a Twisted private interface...
+            self.assertEqual(target._tls._hostname, "meejah.ca")
             self.assertEqual(target._host, u'meejah.ca')
             self.assertEqual(target._port, 443)
             return defer.succeed(proto)
@@ -107,3 +155,34 @@ def getConnection(key, endpoint):
         # apart from the getConnection asserts...
         res = yield agent.request(b'GET', b'https://meejah.ca')
         self.assertIs(res, gold)
+
+    @defer.inlineCallbacks
+    def test_agent_with_circuit_tls_context_factory(self):
+        reactor = Mock()
+        circuit = Mock()
+        socks_ep = Mock()
+        proto = Mock()
+        gold = object()
+        proto.request = Mock(return_value=defer.succeed(gold))
+
+        def getConnection(key, endpoint):
+            target = endpoint._target_endpoint
+            self.assertIsInstance(
+                target._tls,
+                BrowserLikePolicyForHTTPS().creatorForNetloc(b"host", 443).__class__
+            )
+            # This uses a Twisted private interface...
+            self.assertEqual(target._tls._hostname, "custom.domain")
+            self.assertEqual(target._host, 'meejah.ca')
+            return defer.succeed(proto)
+        pool = Mock()
+        pool.getConnection = getConnection
+
+        agent = yield tor_agent(
+            reactor, socks_ep, circuit=circuit, pool=pool,
+            tls_context_factory=CustomTLSContextFactory()
+        )
+
+        # apart from the getConnection asserts...
+        res = yield agent.request(b'GET', b'https://meejah.ca')
+        self.assertIs(res, gold)

txtorcon/circuit.py

@@ -264,13 +264,16 @@ def when_closed(self):
             return defer.succeed(self)
         return self._when_closed.when_fired()
 
-    def web_agent(self, reactor, socks_endpoint, pool=None):
+    def web_agent(self, reactor, socks_endpoint, pool=None, tls_context_factory=None):
         """
         :param socks_endpoint: create one with
             :meth:`txtorcon.TorConfig.create_socks_endpoint`. Can be a
             Deferred.
 
         :param pool: passed on to the Agent (as ``pool=``)
+
+        :param tls_context_factory: A factory for TLS contexts. If ``None``,
+            ``BrowserLikePolicyForHTTPS`` is used.
         """
         # local import because there isn't Agent stuff on some
         # platforms we support, so this will only error if you try
@@ -281,6 +284,7 @@ def web_agent(self, reactor, socks_endpoint, pool=None):
             socks_endpoint,
             circuit=self,
             pool=pool,
+            tls_context_factory=tls_context_factory,
         )
 
     # XXX should make this API match above web_agent (i.e. pass a

txtorcon/controller.py

@@ -566,7 +566,7 @@ def get_config(self):
             self._config = yield TorConfig.from_protocol(self._protocol)
         returnValue(self._config)
 
-    def web_agent(self, pool=None, socks_endpoint=None):
+    def web_agent(self, pool=None, socks_endpoint=None, tls_context_factory=None):
         """
         :param socks_endpoint: If ``None`` (the default), a suitable
             SOCKS port is chosen from our config (or added). If supplied,
@@ -577,6 +577,9 @@ def web_agent(self, pool=None, socks_endpoint=None):
             this.
 
         :param pool: passed on to the Agent (as ``pool=``)
+
+        :param tls_context_factory: A factory for TLS contexts. If ``None``,
+            ``BrowserLikePolicyForHTTPS`` is used.
         """
         if self._non_anonymous:
             raise Exception(
@@ -597,6 +600,7 @@ def web_agent(self, pool=None, socks_endpoint=None):
             self._reactor,
             socks_endpoint,
             pool=pool,
+            tls_context_factory=tls_context_factory
         )
 
     @inlineCallbacks

txtorcon/interface.py

@@ -41,9 +41,9 @@ def create_state(self):
         :class:`txtorcon.TorState` instance.
         """
 
-    def web_agent(self, pool=None, _socks_endpoint=None):
+    def web_agent(self, pool=None, socks_endpoint=None, tls_context_factory=None):
         """
-        :param _socks_endpoint: If ``None`` (the default), a suitable
+        :param socks_endpoint: If ``None`` (the default), a suitable
             SOCKS port is chosen from our config (or added). If supplied,
             should be a Deferred which fires an IStreamClientEndpoint
             (e.g. the return-value from
@@ -52,6 +52,9 @@ def web_agent(self, pool=None, _socks_endpoint=None):
             this.
 
         :param pool: passed on to the Agent (as ``pool=``)
+
+        :param tls_context_factory: A factory for TLS contexts. If ``None``,
+            ``BrowserLikePolicyForHTTPS`` is used.
         """
 
     def dns_resolve(self, hostname):

txtorcon/web.py

@@ -5,7 +5,7 @@
 from __future__ import with_statement
 
 from twisted.web.iweb import IAgentEndpointFactory
-from twisted.web.client import Agent
+from twisted.web.client import Agent, BrowserLikePolicyForHTTPS
 from twisted.internet.defer import inlineCallbacks, returnValue, Deferred
 from twisted.internet.endpoints import TCP4ClientEndpoint, UNIXClientEndpoint
 
@@ -18,7 +18,7 @@
 
 @implementer(IAgentEndpointFactory)
 class _AgentEndpointFactoryUsingTor(object):
-    def __init__(self, reactor, tor_socks_endpoint):
+    def __init__(self, reactor, tor_socks_endpoint, tls_context_factory):
         self._reactor = reactor
         self._proxy_ep = SingleObserver()
         # if _proxy_ep is Deferred, but we get called twice, we must
@@ -28,40 +28,55 @@ def __init__(self, reactor, tor_socks_endpoint):
         else:
             self._proxy_ep.fire(tor_socks_endpoint)
 
+        if tls_context_factory is None:
+            tls_context_factory = BrowserLikePolicyForHTTPS()
+        self._tls_context_factory = tls_context_factory
+
     def _set_proxy(self, p):
         self._proxy_ep.fire(p)
         return p
 
     def endpointForURI(self, uri):
+        if uri.scheme == b'https':
+            tls = self._tls_context_factory.creatorForNetloc(uri.host, uri.port)
+        else:
+            tls = False
         return TorSocksEndpoint(
             self._proxy_ep.when_fired(),
             uri.host,
             uri.port,
-            tls=(uri.scheme == b'https'),
+            tls=tls,
         )
 
 
 @implementer(IAgentEndpointFactory)
 class _AgentEndpointFactoryForCircuit(object):
-    def __init__(self, reactor, tor_socks_endpoint, circ):
+    def __init__(self, reactor, tor_socks_endpoint, circ, tls_context_factory):
         self._reactor = reactor
         self._socks_ep = tor_socks_endpoint
         self._circ = circ
+        if tls_context_factory is None:
+            tls_context_factory = BrowserLikePolicyForHTTPS()
+        self._tls_context_factory = tls_context_factory
 
     def endpointForURI(self, uri):
         """IAgentEndpointFactory API"""
+        if uri.scheme == b'https':
+            tls = self._tls_context_factory.creatorForNetloc(uri.host, uri.port)
+        else:
+            tls = False
         torsocks = TorSocksEndpoint(
             self._socks_ep,
             uri.host, uri.port,
-            tls=uri.scheme == b'https',
+            tls=tls,
         )
         from txtorcon.circuit import TorCircuitEndpoint
         return TorCircuitEndpoint(
             self._reactor, self._circ._torstate, self._circ, torsocks,
         )
 
 
-def tor_agent(reactor, socks_endpoint, circuit=None, pool=None):
+def tor_agent(reactor, socks_endpoint, circuit=None, pool=None, tls_context_factory=None):
     """
     This is the low-level method used by
     :meth:`txtorcon.Tor.web_agent` and
@@ -83,21 +98,29 @@ def tor_agent(reactor, socks_endpoint, circuit=None, pool=None):
         which points at a SOCKS5 port of our Tor
 
     :param pool: passed on to the Agent (as ``pool=``)
-    """
 
+    :param tls_context_factory: A factory for TLS contexts. If ``None``,
+        ``BrowserLikePolicyForHTTPS`` is used.
+    """
     if socks_endpoint is None:
         raise ValueError(
             "Must provide socks_endpoint as Deferred or IStreamClientEndpoint"
         )
     if circuit is not None:
-        factory = _AgentEndpointFactoryForCircuit(reactor, socks_endpoint, circuit)
+        factory = _AgentEndpointFactoryForCircuit(
+            reactor, socks_endpoint, circuit, tls_context_factory
+        )
     else:
-        factory = _AgentEndpointFactoryUsingTor(reactor, socks_endpoint)
+        factory = _AgentEndpointFactoryUsingTor(
+            reactor, socks_endpoint, tls_context_factory
+        )
+
     return Agent.usingEndpointFactory(reactor, factory, pool=pool)
 
 
 @inlineCallbacks
-def agent_for_socks_port(reactor, torconfig, socks_config, pool=None):
+def agent_for_socks_port(reactor, torconfig, socks_config, pool=None,
+                         tls_context_factory=None):
     """
     This returns a Deferred that fires with an object that implements
     :class:`twisted.web.iweb.IAgent` and is thus suitable for passing
@@ -116,13 +139,10 @@ def agent_for_socks_port(reactor, torconfig, socks_config, pool=None):
         containing ``socket``). If the given SOCKS option is not
         already available in the underlying Tor instance, it is
         re-configured to add the SOCKS option.
-    """
-    # :param tls: True (the default) will use Twisted's default options
-    #     with the hostname in the URI -- that is, TLS verification
-    #     similar to a Browser. Otherwise, you can pass whatever Twisted
-    #     returns for `optionsForClientTLS
-    #     <https://twistedmatrix.com/documents/current/api/twisted.internet.ssl.optionsForClientTLS.html>`_
 
+    :param tls_context_factory: A factory for TLS contexts. If ``None``,
+        ``BrowserLikePolicyForHTTPS`` is used.
+    """
     socks_config = str(socks_config)  # sadly, all lists are lists-of-strings to Tor :/
     if socks_config not in torconfig.SocksPort:
         txtorlog.msg("Adding SOCKS port '{}' to Tor".format(socks_config))
@@ -149,7 +169,9 @@ def agent_for_socks_port(reactor, torconfig, socks_config, pool=None):
     returnValue(
         Agent.usingEndpointFactory(
             reactor,
-            _AgentEndpointFactoryUsingTor(reactor, socks_ep),
+            _AgentEndpointFactoryUsingTor(
+                reactor, socks_ep, tls_context_factory=tls_context_factory
+            ),
             pool=pool,
         )
     )