Failing tests without network connection

[dotlambda]

Long story short

I get 2 failing tests without a network connection:

• test_request_tracing
• test_request_tracing_interpose_headers

Expected behaviour

No failing tests.

Actual behaviour

=================================== FAILURES ===================================
_________________________ test_request_tracing[pyloop] _________________________

self = <aiohttp.connector.TCPConnector object at 0x7fffeecdc470>
req = <aiohttp.client_reqrep.ClientRequest object at 0x7fffeece0a58>

    async def _create_direct_connection(self, req,
                                        *, client_error=ClientConnectorError,
                                        traces=None):
        sslcontext = self._get_ssl_context(req)
        fingerprint = self._get_fingerprint(req)

        try:
            hosts = await self._resolve_host(
                req.url.raw_host,
                req.port,
>               traces=traces)

aiohttp/connector.py:819:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <aiohttp.connector.TCPConnector object at 0x7fffeecdc470>
host = 'example.com', port = 80, traces = None

    async def _resolve_host(self, host, port, traces=None):
        if is_ip_address(host):
            return [{'hostname': host, 'host': host, 'port': port,
                     'family': self._family, 'proto': 0, 'flags': 0}]

        if not self._use_dns_cache:

            if traces:
                for trace in traces:
                    await trace.send_dns_resolvehost_start(host)

            res = (await self._resolver.resolve(
                host, port, family=self._family))

            if traces:
                for trace in traces:
                    await trace.send_dns_resolvehost_end(host)

            return res

        key = (host, port)

        if (key in self._cached_hosts) and \
                (not self._cached_hosts.expired(key)):

            if traces:
                for trace in traces:
                    await trace.send_dns_cache_hit(host)

            return self._cached_hosts.next_addrs(key)

        if key in self._throttle_dns_events:
            if traces:
                for trace in traces:
                    await trace.send_dns_cache_hit(host)
            await self._throttle_dns_events[key].wait()
        else:
            if traces:
                for trace in traces:
                    await trace.send_dns_cache_miss(host)
            self._throttle_dns_events[key] = \
                EventResultOrError(self._loop)
            try:

                if traces:
                    for trace in traces:
                        await trace.send_dns_resolvehost_start(host)

                addrs = await \
                    asyncio.shield(self._resolver.resolve(host,
                                                          port,
                                                          family=self._family),
>                                  loop=self._loop)

aiohttp/connector.py:706:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <aiohttp.resolver.ThreadedResolver object at 0x7fffeecdc9b0>
host = 'example.com', port = 80, family = 0

    async def resolve(self, host, port=0, family=socket.AF_INET):
        infos = await self._loop.getaddrinfo(
>           host, port, type=socket.SOCK_STREAM, family=family)

aiohttp/resolver.py:30:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = None

    def run(self):
        if not self.future.set_running_or_notify_cancel():
            return

        try:
>           result = self.fn(*self.args, **self.kwargs)

/nix/store/87wc186a9xq5y6yxvar6f2sm5cbqzpzr-python3-3.6.4/lib/python3.6/concurrent/futures/thread.py:56:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

host = 'example.com', port = 80, family = 0, type = <SocketKind.SOCK_STREAM: 1>
proto = 0, flags = 0

    def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
        """Resolve host and port into list of address info entries.

        Translate the host/port argument into a sequence of 5-tuples that contain
        all the necessary arguments for creating a socket connected to that service.
        host is a domain name, a string representation of an IPv4/v6 address or
        None. port is a string service name such as 'http', a numeric port number or
        None. By passing None as the value of host and port, you can pass NULL to
        the underlying C API.

        The family, type and proto arguments can be optionally specified in order to
        narrow the list of addresses returned. Passing zero as a value for each of
        these arguments selects the full range of results.
        """
        # We override this function since we want to translate the numeric family
        # and socket type values to enum constants.
        addrlist = []
>       for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
E       socket.gaierror: [Errno -2] Name or service not known

/nix/store/87wc186a9xq5y6yxvar6f2sm5cbqzpzr-python3-3.6.4/lib/python3.6/socket.py:745: gaierror

The above exception was the direct cause of the following exception:

loop = <_UnixSelectorEventLoop running=False closed=False debug=False>

    async def test_request_tracing(loop):
        trace_config_ctx = mock.Mock()
        trace_request_ctx = {}
        on_request_start = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
        on_request_redirect = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))
        on_request_end = mock.Mock(side_effect=asyncio.coroutine(mock.Mock()))

        trace_config = aiohttp.TraceConfig(
            trace_config_ctx_factory=mock.Mock(return_value=trace_config_ctx)
        )
        trace_config.on_request_start.append(on_request_start)
        trace_config.on_request_end.append(on_request_end)
        trace_config.on_request_redirect.append(on_request_redirect)

        async with aiohttp.ClientSession(loop=loop,
                                         trace_configs=[trace_config]) as session:

            async with await session.get(
                    'http://example.com',
>                   trace_request_ctx=trace_request_ctx) as resp:

tests/test_client_session.py:477:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
aiohttp/client.py:319: in _request
    traces=traces
aiohttp/connector.py:418: in connect
    traces=traces
aiohttp/connector.py:736: in _create_connection
    traces=None
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <aiohttp.connector.TCPConnector object at 0x7fffeecdc470>
req = <aiohttp.client_reqrep.ClientRequest object at 0x7fffeece0a58>

    async def _create_direct_connection(self, req,
                                        *, client_error=ClientConnectorError,
                                        traces=None):
        sslcontext = self._get_ssl_context(req)
        fingerprint = self._get_fingerprint(req)

        try:
            hosts = await self._resolve_host(
                req.url.raw_host,
                req.port,
                traces=traces)
        except OSError as exc:
            # in case of proxy it is not ClientProxyConnectionError
            # it is problem of resolving proxy ip itself
>           raise ClientConnectorError(req.connection_key, exc) from exc
E           aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host example.com:80 ssl:False [Name or service not known]

aiohttp/connector.py:823: ClientConnectorError
________________ test_request_tracing_interpose_headers[pyloop] ________________

self = <aiohttp.connector.TCPConnector object at 0x7fffeec13da0>
req = <test_client_session.test_request_tracing_interpose_headers.<locals>.MyClientRequest object at 0x7fffeec137b8>

    async def _create_direct_connection(self, req,
                                        *, client_error=ClientConnectorError,
                                        traces=None):
        sslcontext = self._get_ssl_context(req)
        fingerprint = self._get_fingerprint(req)

        try:
            hosts = await self._resolve_host(
                req.url.raw_host,
                req.port,
>               traces=traces)

aiohttp/connector.py:819:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <aiohttp.connector.TCPConnector object at 0x7fffeec13da0>
host = 'example.com', port = 80, traces = None

    async def _resolve_host(self, host, port, traces=None):
        if is_ip_address(host):
            return [{'hostname': host, 'host': host, 'port': port,
                     'family': self._family, 'proto': 0, 'flags': 0}]

        if not self._use_dns_cache:

            if traces:
                for trace in traces:
                    await trace.send_dns_resolvehost_start(host)

            res = (await self._resolver.resolve(
                host, port, family=self._family))

            if traces:
                for trace in traces:
                    await trace.send_dns_resolvehost_end(host)

            return res

        key = (host, port)

        if (key in self._cached_hosts) and \
                (not self._cached_hosts.expired(key)):

            if traces:
                for trace in traces:
                    await trace.send_dns_cache_hit(host)

            return self._cached_hosts.next_addrs(key)

        if key in self._throttle_dns_events:
            if traces:
                for trace in traces:
                    await trace.send_dns_cache_hit(host)
            await self._throttle_dns_events[key].wait()
        else:
            if traces:
                for trace in traces:
                    await trace.send_dns_cache_miss(host)
            self._throttle_dns_events[key] = \
                EventResultOrError(self._loop)
            try:

                if traces:
                    for trace in traces:
                        await trace.send_dns_resolvehost_start(host)

                addrs = await \
                    asyncio.shield(self._resolver.resolve(host,
                                                          port,
                                                          family=self._family),
>                                  loop=self._loop)

aiohttp/connector.py:706:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <aiohttp.resolver.ThreadedResolver object at 0x7fffeec13eb8>
host = 'example.com', port = 80, family = 0

    async def resolve(self, host, port=0, family=socket.AF_INET):
        infos = await self._loop.getaddrinfo(
>           host, port, type=socket.SOCK_STREAM, family=family)

aiohttp/resolver.py:30:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = None

    def run(self):
        if not self.future.set_running_or_notify_cancel():
            return

        try:
>           result = self.fn(*self.args, **self.kwargs)

/nix/store/87wc186a9xq5y6yxvar6f2sm5cbqzpzr-python3-3.6.4/lib/python3.6/concurrent/futures/thread.py:56:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

host = 'example.com', port = 80, family = 0, type = <SocketKind.SOCK_STREAM: 1>
proto = 0, flags = 0

    def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
        """Resolve host and port into list of address info entries.

        Translate the host/port argument into a sequence of 5-tuples that contain
        all the necessary arguments for creating a socket connected to that service.
        host is a domain name, a string representation of an IPv4/v6 address or
        None. port is a string service name such as 'http', a numeric port number or
        None. By passing None as the value of host and port, you can pass NULL to
        the underlying C API.

        The family, type and proto arguments can be optionally specified in order to
        narrow the list of addresses returned. Passing zero as a value for each of
        these arguments selects the full range of results.
        """
        # We override this function since we want to translate the numeric family
        # and socket type values to enum constants.
        addrlist = []
>       for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
E       socket.gaierror: [Errno -2] Name or service not known

/nix/store/87wc186a9xq5y6yxvar6f2sm5cbqzpzr-python3-3.6.4/lib/python3.6/socket.py:745: gaierror

The above exception was the direct cause of the following exception:

loop = <_UnixSelectorEventLoop running=False closed=False debug=False>

    async def test_request_tracing_interpose_headers(loop):

        class MyClientRequest(ClientRequest):
            headers = None

            def __init__(self, *args, **kwargs):
                super(MyClientRequest, self).__init__(*args, **kwargs)
                MyClientRequest.headers = self.headers

        async def new_headers(
                session,
                trace_config_ctx,
                data):
            data.headers['foo'] = 'bar'

        trace_config = aiohttp.TraceConfig()
        trace_config.on_request_start.append(new_headers)

        session = aiohttp.ClientSession(
            loop=loop,
            request_class=MyClientRequest,
            trace_configs=[trace_config]
        )

>       await session.get('http://example.com')

tests/test_client_session.py:565:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
aiohttp/client.py:319: in _request
    traces=traces
aiohttp/connector.py:418: in connect
    traces=traces
aiohttp/connector.py:736: in _create_connection
    traces=None
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <aiohttp.connector.TCPConnector object at 0x7fffeec13da0>
req = <test_client_session.test_request_tracing_interpose_headers.<locals>.MyClientRequest object at 0x7fffeec137b8>

    async def _create_direct_connection(self, req,
                                        *, client_error=ClientConnectorError,
                                        traces=None):
        sslcontext = self._get_ssl_context(req)
        fingerprint = self._get_fingerprint(req)

        try:
            hosts = await self._resolve_host(
                req.url.raw_host,
                req.port,
                traces=traces)
        except OSError as exc:
            # in case of proxy it is not ClientProxyConnectionError
            # it is problem of resolving proxy ip itself
>           raise ClientConnectorError(req.connection_key, exc) from exc
E           aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host example.com:80 ssl:False [Name or service not known]

aiohttp/connector.py:823: ClientConnectorError
======== 2 failed, 2038 passed, 24 skipped, 4 xfailed in 57.09 seconds =========

Steps to reproduce

git clone https://github.com/dotlambda/nixpkgs.git
cd nixpkgs
git checkout aiohttp
nix-build -A python3.pkgs.aiohttp

Your environment

aiohttp-3.0.0 on NixOS
Nix builds derivations in a sandbox which disallows any network access.
You can find the Nix derivation I use here: https://github.com/dotlambda/nixpkgs/blob/aiohttp/pkgs/development/python-modules/aiohttp/default.nix

[asvetlov]

Thanks for report!

[dotlambda]

Thanks a lot for your quick reaction!

[lock]

This thread has been automatically locked since there has not been any recent activity after it was closed. Please open a [new issue] for related bugs.
If you feel like there's important points made in this discussion, please include those exceprts into that [new issue].
[new issue]: https://github.com/aio-libs/aiohttp/issues/new