Ensure import urllib3 can succeed even if trio and twisted aren't installed

test/test_backends.py

@@ -0,0 +1,68 @@
+import sys
+
+import pytest
+
+import urllib3
+from urllib3.backends import Backend
+from urllib3._backends._loader import normalize_backend, load_backend
+
+
+requires_async_pool_manager = pytest.mark.skipif(
+    not hasattr(urllib3, "AsyncPoolManager"),
+    reason="async backends require AsyncPoolManager",
+)
+
+
+requires_sync_pool_manager = pytest.mark.skipif(
+    hasattr(urllib3, "AsyncPoolManager"),
+    reason="sync backends cannot be used with AsyncPoolManager",
+)
+
+
+class TestNormalizeBackend(object):
+    """
+    Assert that we fail correctly if we attempt to use an unknown or incompatible backend.
+    """
+    def test_unknown(self):
+        with pytest.raises(ValueError) as excinfo:
+            normalize_backend("_unknown")
+
+        assert 'unknown backend specifier _unknown' == str(excinfo.value)
+
+    @requires_sync_pool_manager
+    def test_sync(self):
+        assert normalize_backend(Backend("sync")) == Backend("sync")
+        assert normalize_backend("sync") == Backend("sync")
+        assert normalize_backend(None) == Backend("sync")
+
+        with pytest.raises(ValueError) as excinfo:
+            normalize_backend(Backend("trio"))
+
+        assert 'trio backend requires urllib3 to be built with async support' == str(excinfo.value)
+
+    @requires_async_pool_manager
+    def test_async(self):
+        assert normalize_backend(Backend("trio")) == Backend("trio")
+        assert normalize_backend("twisted") == Backend("twisted")
+
+        with pytest.raises(ValueError) as excinfo:
+            normalize_backend(Backend("sync"))
+
+        assert 'sync backend requires urllib3 to be built without async support' == str(excinfo.value)
+
+        from twisted.internet import reactor
+        assert normalize_backend(Backend("twisted", reactor=reactor)) == Backend("twisted", reactor=reactor)
+
+
+class TestLoadBackend(object):
+    """
+    Assert that we can load a normalized backend
+    """
+    @requires_sync_pool_manager()
+    def test_sync(self):
+        load_backend(normalize_backend("sync"))
+
+    @requires_async_pool_manager()
+    def test_async(self):
+        from twisted.internet import reactor
+        load_backend(Backend("twisted", reactor=reactor))

test/test_no_ssl.py

@@ -90,6 +90,5 @@ def import_ssl():
 
         self.assertRaises(ImportError, import_ssl)
 
-    @pytest.mark.xfail
     def test_import_urllib3(self):
         import urllib3  # noqa: F401

test/with_dummyserver/test_no_ssl.py

@@ -8,10 +8,17 @@
 from dummyserver.testcase import (
         HTTPDummyServerTestCase, HTTPSDummyServerTestCase)
 
+import pytest
 import urllib3
 
 
 class TestHTTPWithoutSSL(HTTPDummyServerTestCase, TestWithoutSSL):
+
+    @pytest.mark.skip(reason=(
+        "TestWithoutSSL mutates sys.modules."
+        "This breaks the backend loading code which imports modules at runtime."
+        "See discussion at https://github.com/python-trio/urllib3/pull/42"
+    ))
     def test_simple(self):
         pool = urllib3.HTTPConnectionPool(self.host, self.port)
         self.addCleanup(pool.close)

urllib3/_async/connection.py

@@ -30,8 +30,8 @@
 )
 from urllib3.packages import six
 from ..util import ssl_ as ssl_util
-from .._backends import SyncBackend
 from .._backends._common import LoopAbort
+from .._backends._loader import load_backend
 
 try:
     import ssl
@@ -311,8 +311,7 @@ def __init__(self, host, port, backend=None,
                  source_address=None, tunnel_host=None, tunnel_port=None,
                  tunnel_headers=None):
         self.is_verified = False
-
-        self._backend = backend or SyncBackend()
+        self._backend = backend or load_backend("sync")
         self._host = host
         self._port = port
         self._socket_options = (

urllib3/_backends/__init__.py

@@ -1,9 +0,0 @@
-from urllib3.packages import six
-from .sync_backend import SyncBackend
-
-__all__ = ['SyncBackend']
-
-if six.PY3:
-    from .trio_backend import TrioBackend
-    from .twisted_backend import TwistedBackend
-    __all__ += ['TrioBackend', 'TwistedBackend']

urllib3/_backends/_loader.py

@@ -0,0 +1,102 @@
+import sys
+
+from ..backends import Backend
+
+
+class Loader:
+
+    def __init__(self, name, loader, is_async):
+        self.name = name
+        self.loader = loader
+        self.is_async = is_async
+
+    def __call__(self, *args, **kwargs):
+        return self.loader(kwargs)
+
+
+def load_sync_backend(kwargs):
+    from .sync_backend import SyncBackend
+    return SyncBackend(**kwargs)
+
+
+def load_trio_backend(kwargs):
+    from .trio_backend import TrioBackend
+    return TrioBackend(**kwargs)
+
+
+def load_twisted_backend(kwargs):
+    from .twisted_backend import TwistedBackend
+    return TwistedBackend(**kwargs)
+
+
+def backend_directory():
+    """
+    We defer any heavy duty imports until the last minute.
+    """
+    loaders = [
+        Loader(
+            name="sync",
+            loader=load_sync_backend,
+            is_async=False,
+        ),
+        Loader(
+            name="trio",
+            loader=load_trio_backend,
+            is_async=True,
+        ),
+        Loader(
+            name="twisted",
+            loader=load_twisted_backend,
+            is_async=True,
+        ),
+    ]
+    return {
+        loader.name: loader for loader in loaders
+    }
+
+
+def async_supported():
+    """
+    Tests if the async keyword is supported.
+    """
+    async def f():
+        """
+        Functions with an `async` prefix return a coroutine.
+        This is removed by the bleaching code, which will change this function to return None.
+        """
+        return None
+
+    obj = f()
+    if obj is None:
+        return False
+    else:
+        obj.close()  # prevent unawaited coroutine warning
+        return True
+
+
+def normalize_backend(backend):
+    if backend is None:
+        backend = Backend(name="sync")  # sync backend is the default
+    elif not isinstance(backend, Backend):
+        backend = Backend(name=backend)
+
+    loaders_by_name = backend_directory()
+    if backend.name not in loaders_by_name:
+        raise ValueError("unknown backend specifier {}".format(backend.name))
+
+    loader = loaders_by_name[backend.name]
+
+    is_async_supported = async_supported()
+    if is_async_supported and not loader.is_async:
+        raise ValueError("{} backend requires urllib3 to be built without async support".format(loader.name))
+
+    if not is_async_supported and loader.is_async:
+        raise ValueError("{} backend requires urllib3 to be built with async support".format(loader.name))
+
+    return backend
+
+
+def load_backend(backend):
+    loaders_by_name = backend_directory()
+    loader = loaders_by_name[backend.name]
+    return loader(backend.kwargs)

urllib3/_backends/twisted_backend.py

@@ -1,6 +1,6 @@
 import socket
 import OpenSSL.crypto
-from twisted.internet import protocol, ssl
+from twisted.internet import protocol, reactor as default_reactor, ssl
 from twisted.internet.interfaces import IHandshakeListener
 from twisted.internet.endpoints import HostnameEndpoint, connectProtocol
 from twisted.internet.defer import (
@@ -15,8 +15,8 @@
 
 
 class TwistedBackend:
-    def __init__(self, reactor):
-        self._reactor = reactor
+    def __init__(self, reactor=None):
+        self._reactor = reactor or default_reactor
 
     async def connect(self, host, port, connect_timeout,
                       source_address=None, socket_options=None):

urllib3/backends.py

@@ -0,0 +1,12 @@
+class Backend:
+    """
+    Specifies the desired backend and any arguments passed to its constructor.
+
+    Projects that use urllib3 can subclass this interface to expose it to users.
+    """
+    def __init__(self, name, **kwargs):
+        self.name = name
+        self.kwargs = kwargs
+
+    def __eq__(self, other):
+        return self.name == other.name and self.kwargs == other.kwargs