Add a Tor connection-hint handler

Still preliminary, but this adds `foolscap.connections.tor`, with
functions that build a handler for:

* pre-existing system Tor, over a SOCKS port
* system Tor, over a control port
* launch a new Tor, with optional persistent data directory

The tor.default_socks() should Just Work for systems that:

* have a debian/ubuntu 'tor' package installed
* are currently running a copy of the Tor Browser Bundle

When installing `magic-wormhole[tor]`, this adds a dependency on
txtorcon >= 0.15.0 . A future version of txtorcon will probably have a
better API, so we'll increment the dependency when it becomes available.

setup.py

@@ -61,8 +61,9 @@ def run(self):
     "cmdclass": commands,
     "install_requires": ["twisted[tls] >= 16.0.0", "pyOpenSSL"],
     "extras_require": {
-        "dev": ["mock", "txsocksx"],
+        "dev": ["mock", "txsocksx", "txtorcon >= 0.15.0"],
         "socks": ["txsocksx"],
+        "tor": ["txtorcon >= 0.15.0"],
         },
 }
 

src/foolscap/connections/tor.py

@@ -0,0 +1,167 @@
+import os, re
+from twisted.internet.interfaces import IStreamClientEndpoint
+from twisted.internet.defer import inlineCallbacks, returnValue, succeed
+import ipaddress
+from .. import observer
+
+from zope.interface import implementer
+from ..ipb import IConnectionHintHandler, InvalidHintError
+from ..util import allocate_tcp_port
+import txtorcon
+from .tcp import DOTTED_QUAD_RESTR, DNS_NAME_RESTR
+
+def is_non_public_numeric_address(host):
+    # for numeric hostnames, skip RFC1918 addresses, since no Tor exit
+    # node will be able to reach those. Likewise ignore IPv6 addresses.
+    try:
+        a = ipaddress.ip_address(host.decode("ascii")) # wants unicode
+    except ValueError:
+        return False # non-numeric, let Tor try it
+    if a.version != 4:
+        return True # IPv6 gets ignored
+    if (a.is_loopback or a.is_multicast or a.is_private or a.is_reserved
+        or a.is_unspecified):
+        return True # too weird, don't connect
+    return False
+
+HINT_RE = re.compile(r"^[^:]*:(%s|%s):(\d+){1,5}$" % (DOTTED_QUAD_RESTR,
+                                                      DNS_NAME_RESTR))
+
+@implementer(IConnectionHintHandler)
+class _Common:
+    # subclasses must:
+    #  define _connect()
+    #  set self._socks_hostname, self._socks_port
+
+    def __init__(self):
+        self._connected = False
+        self._when_connected = observer.OneShotObserverList()
+
+    def _maybe_connect(self, reactor):
+        if not self._connected:
+            self._connected = True
+            # connect
+            d = self._connect()
+            d.addBoth(self._when_connected.fire)
+        return self._when_connected.whenFired()
+
+    @inlineCallbacks
+    def hint_to_endpoint(self, hint, reactor):
+        # Return (endpoint, hostname), where "hostname" is what we pass to the
+        # HTTP "Host:" header so a dumb HTTP server can be used to redirect us.
+        mo = HINT_RE.search(hint)
+        if not mo:
+            raise InvalidHintError("unrecognized TCP/Tor hint")
+        host, port = mo.group(1), int(mo.group(2))
+        if is_non_public_numeric_address(host):
+            raise InvalidHintError("ignoring non-Tor-able ipaddr %s" % host)
+        yield self._maybe_connect(reactor)
+        # txsocksx doesn't like unicode: it concatenates some binary protocol
+        # bytes with the hostname when talking to the SOCKS server, so the
+        # py2 automatic unicode promotion blows up
+        host = host.encode("ascii")
+        ep = txtorcon.TorClientEndpoint(host, port,
+                                        socks_hostname=self._socks_hostname,
+                                        socks_port=self._socks_port)
+        returnValue( (ep, host) )
+
+
+# note: TorClientEndpoint imports 'reactor' itself, doesn't provide override
+
+class _SocksTor(_Common):
+    def __init__(self, hostname=None, port=None):
+        _Common.__init__(self)
+        self._connnected = True # no need to call _connect()
+        self._socks_hostname = hostname
+        self._socks_port = port # None means to use defaults: 9050, then 9150
+    def _connect(self):
+        return succeed(None)
+
+def default_socks():
+    # TorClientEndpoint knows how to cycle through a built-in set of socks
+    # ports, but it doesn't know to set the hostname to localhost
+    return _SocksTor("127.0.0.1")
+
+def socks_on_port(port):
+    return _SocksTor("127.0.0.1", port)
+
+
+class _LaunchedTor(_Common):
+    def __init__(self, reactor, data_directory=None, tor_binary=None):
+        _Common.__init__(self)
+        self._reactor = reactor
+        self._data_directory = data_directory
+        self._tor_binary = tor_binary
+
+    @inlineCallbacks
+    def _connect(self):
+        # create a new Tor
+        config = self.config = txtorcon.TorConfig()
+        if self._data_directory:
+            # The default is for launch_tor to create a tempdir itself, and
+            # delete it when done. We only need to set a DataDirectory if we
+            # want it to be persistent. This saves some startup time, because
+            # we cache the descriptors from last time. On one of my hosts,
+            # this reduces connect from 20s to 15s.
+            if not os.path.exists(self._data_directory):
+                # tor will mkdir this, but txtorcon wants to chdir to it
+                # before spawning the tor process, so (for now) we need to
+                # mkdir it ourselves. TODO: txtorcon should take
+                # responsibility for this.
+                os.mkdir(self._data_directory)
+            config.DataDirectory = self._data_directory
+
+        #config.ControlPort = allocate_tcp_port() # defaults to 9052
+        config.SocksPort = allocate_tcp_port()
+        self._socks_hostname = "127.0.0.1"
+        self._socks_port = config.SocksPort
+
+        #print "launching tor"
+        tpp = yield txtorcon.launch_tor(config, self._reactor,
+                                        tor_binary=self._tor_binary,
+                                        )
+        #print "launched"
+        # gives a TorProcessProtocol with .tor_protocol
+        self._tor_protocol = tpp.tor_protocol
+        returnValue(True)
+
+def launch_tor(reactor, data_directory=None, tor_binary=None):
+    """Return a handler which launches a new Tor process (once).
+    - data_directory: a persistent directory where Tor can cache its
+      descriptors. This allows subsequent invocations to start faster. If
+      None, the process will use an ephemeral tempdir, deleting it when Tor
+      exits.
+    - tor_binary: the path to the Tor executable we should use. If None,
+      search $PATH.
+    """
+    return _LaunchedTor(reactor, data_directory, tor_binary)
+
+
+@implementer(IConnectionHintHandler)
+class _ConnectedTor(_Common):
+    def __init__(self, reactor, tor_control_endpoint):
+        _Common.__init__(self)
+        self._reactor = reactor
+        assert IStreamClientEndpoint.providedBy(tor_control_endpoint)
+        self._tor_control_endpoint = tor_control_endpoint
+
+    @inlineCallbacks
+    def _connect(self):
+        ep = self._tor_control_endpoint
+        tproto = yield txtorcon.build_tor_connection(ep, build_state=False)
+        config = yield txtorcon.TorConfig.from_protocol(tproto)
+        ports = list(config.SocksPort)
+        port = ports[0]
+        if port == "DEFAULT":
+            port = "9050"
+        port = int(port)
+        self._socks_hostname = "127.0.0.1"
+        self._socks_port = port
+
+
+def with_control_port(reactor, tor_control_endpoint):
+    """Return a handler which connects to a pre-existing Tor process on the
+    given control port.
+    - control_port: a ClientEndpoint which points at the Tor control port
+    """
+    return _ConnectedTor(reactor, tor_control_endpoint)

src/foolscap/test/check-connections-client.py

@@ -14,31 +14,12 @@
 # Then run 'check-connections-client.py tcp', then with 'socks', then with
 # 'tor'.
 
-import sys
+import os, sys, time
 from twisted.internet import reactor
+from twisted.internet.defer import inlineCallbacks
+from twisted.internet.endpoints import HostnameEndpoint, clientFromString
 from foolscap.api import Referenceable, Tub
 
-class Observer(Referenceable):
-    def remote_event(self, msg):
-        print "event:", msg
-
-def printResult(number):
-    print "the result is", number
-def gotError(err):
-    print "got an error:", err
-def gotRemote(remote):
-    o = Observer()
-    d = remote.callRemote("addObserver", observer=o)
-    d.addCallback(lambda res: remote.callRemote("push", num=2))
-    d.addCallback(lambda res: remote.callRemote("push", num=3))
-    d.addCallback(lambda res: remote.callRemote("add"))
-    d.addCallback(lambda res: remote.callRemote("pop"))
-    d.addCallback(printResult)
-    d.addCallback(lambda res: remote.callRemote("removeObserver", observer=o))
-    d.addErrback(gotError)
-    d.addCallback(lambda res: reactor.stop())
-    return d
-
 
 tub = Tub()
 
@@ -50,31 +31,78 @@ def gotRemote(remote):
     # which connections will emerge from the other end. Check the server logs
     # to see the peer address of each addObserver call to verify that it is
     # coming from 127.0.0.1 rather than the client host.
-    from twisted.internet import endpoints
     from foolscap.connections import socks
-    h = socks.SOCKS(endpoints.HostnameEndpoint(reactor, "localhost", 8013))
+    h = socks.SOCKS(HostnameEndpoint(reactor, "localhost", 8013))
     tub.removeAllConnectionHintHandlers()
     tub.addConnectionHintHandler("tcp", h)
     furl = "pb://%s@tcp:localhost:%d/calculator" % (TUBID, LOCALPORT)
-elif which == "tor":
-    from twisted.internet import endpoints
+elif which in ("default-socks", "socks-port", "launch-tor", "control-tor"):
     from foolscap.connections import tor
-    h = tor.Tor()
+    if which == "default-socks":
+        h = tor.default_socks()
+    elif which == "socks-port":
+        h = tor.socks_on_port(int(sys.argv[2]))
+    elif which == "launch-tor":
+        data_directory = None
+        if len(sys.argv) > 2:
+            data_directory = os.path.abspath(sys.argv[2])
+        h = tor.launch_tor(reactor, data_directory)
+    elif which == "control-tor":
+        control_ep = clientFromString(reactor, sys.argv[2])
+        h = tor.with_control_port(reactor, control_ep)
     tub.removeAllConnectionHintHandlers()
     tub.addConnectionHintHandler("tor", h)
     furl = "pb://%s@tor:%s:%d/calculator" % (TUBID, ONION, ONIONPORT)
-    print "using tor:", furl
 else:
     print "run as 'check-connections-client.py [tcp|socks|tor]'"
     sys.exit(1)
 print "using %s: %s" % (which, furl)
 
-tub.startService()
-d = tub.getReference(furl)
-d.addCallback(gotRemote)
+class Observer(Referenceable):
+    def remote_event(self, msg):
+        pass
+
+@inlineCallbacks
+def go():
+    tub.startService()
+    start = time.time()
+    rtts = []
+    remote = yield tub.getReference(furl)
+    t_connect = time.time() - start
+
+    o = Observer()
+    start = time.time()
+    yield remote.callRemote("addObserver", observer=o)
+    rtts.append(time.time() - start)
+
+    start = time.time()
+    yield remote.callRemote("removeObserver", observer=o)
+    rtts.append(time.time() - start)
+
+    start = time.time()
+    yield remote.callRemote("push", num=2)
+    rtts.append(time.time() - start)
+
+    start = time.time()
+    yield remote.callRemote("push", num=3)
+    rtts.append(time.time() - start)
+
+    start = time.time()
+    yield remote.callRemote("add")
+    rtts.append(time.time() - start)
+
+    start = time.time()
+    number = yield remote.callRemote("pop")
+    rtts.append(time.time() - start)
+    print "the result is", number
+
+    print "t_connect:", t_connect
+    print "avg rtt:", sum(rtts) / len(rtts)
+
+d = go()
 def _oops(f):
     print "error", f
-    reactor.stop()
 d.addErrback(_oops)
+d.addCallback(lambda res: reactor.stop())
 
 reactor.run()

src/foolscap/test/test_connection.py

@@ -2,13 +2,15 @@
 from zope.interface import implementer
 from twisted.trial import unittest
 from twisted.internet import endpoints, defer, reactor
+from twisted.internet.defer import inlineCallbacks
 from twisted.application import service
 from txsocksx.client import SOCKS5ClientEndpoint
 from foolscap.api import Tub
 from foolscap.connection import get_endpoint
-from foolscap.connections import socks
+from foolscap.connections import socks, tor
 from foolscap.connections.tcp import convert_legacy_hint, DefaultTCP
 from foolscap.tokens import NoLocationHintsError
+from foolscap.ipb import InvalidHintError
 from foolscap.test.common import (certData_low, certData_high, Target,
                                   ShouldFailMixin)
 from foolscap import ipb, util
@@ -213,3 +215,41 @@ def test_bad_hint(self):
                           h.hint_to_endpoint, "tcp:example.com:noport", reactor)
         self.assertRaises(ipb.InvalidHintError,
                           h.hint_to_endpoint, "tcp:@:1234", reactor)
+
+class Tor(unittest.TestCase):
+    @inlineCallbacks
+    def test_default_socks(self):
+        with mock.patch("foolscap.connections.tor.txtorcon") as ttc:
+            ttc.TorClientEndpoint = tce = mock.Mock()
+            tce.return_value = expected_ep = object()
+            h = tor.default_socks()
+            res = yield h.hint_to_endpoint("tcp:example.com:1234", reactor)
+            self.assertEqual(tce.mock_calls,
+                             [mock.call("example.com", 1234,
+                                        socks_hostname="127.0.0.1",
+                                        socks_port=None)])
+            ep, host = res
+            self.assertIdentical(ep, expected_ep)
+            self.assertEqual(host, "example.com")
+
+    def test_badaddr(self):
+        isnon = tor.is_non_public_numeric_address
+        self.assertTrue(isnon("10.0.0.1"))
+        self.assertTrue(isnon("127.0.0.1"))
+        self.assertTrue(isnon("192.168.78.254"))
+        self.assertFalse(isnon("8.8.8.8"))
+        self.assertFalse(isnon("example.org"))
+
+    @inlineCallbacks
+    def test_default_socks_badaddr(self):
+        h = tor.default_socks()
+        d = h.hint_to_endpoint("tcp:10.0.0.1:1234", reactor)
+        f = yield self.assertFailure(d, InvalidHintError)
+        self.assertEqual(str(f), "ignoring non-Tor-able ipaddr 10.0.0.1")
+
+        d = h.hint_to_endpoint("tcp:127.0.0.1:1234", reactor)
+        f = yield self.assertFailure(d, InvalidHintError)
+        self.assertEqual(str(f), "ignoring non-Tor-able ipaddr 127.0.0.1")
+
+    # TODO: exercise launch_tor and with_control_port somehow
+