Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add httpx.NetworkOptions configuration. #3052

Draft
wants to merge 10 commits into
base: master
Choose a base branch
from
44 changes: 44 additions & 0 deletions docs/advanced/network-options.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
There are several advanced network options that are made available through the `httpx.NetworkOptions` configuration class.

```python
# Configure an HTTPTransport with some specific network options.
network_options = httpx.NetworkOptions(
connection_retries=1,
local_address="0.0.0.0",
)
transport = httpx.HTTPTransport(network_options=network_options)

# Instantiate a client with the configured transport.
client = httpx.Client(transport=transport)
```

## Configuration

The options available on this class are...

### `connection_retries`

Configure a number of retries that may be attempted when initially establishing a TCP connection. Defaults to `0`.

### `local_address`

Configure the local address that the socket should be bound too. The most common usage is for enforcing binding to either IPv4 `local_address="0.0.0.0"` or IPv6 `local_address="::"`.

### `socket_options`

Configure the list of socket options to be applied to the underlying sockets used for network connections.
For example, you can use it to explicitly specify which network interface should be used for the connection in this manner:

```python
import httpx

socket_options = [(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, b"ETH999")]

network_options = httpx.NetworkOptions(
socket_options=socket_options
)
```

### `uds`

Connect to a Unix Domain Socket, rather than over the network. Should be a string providing the path to the UDS.
1 change: 1 addition & 0 deletions httpx/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ def main() -> None: # type: ignore
"MockTransport",
"NetRCAuth",
"NetworkError",
"NetworkOptions",
"options",
"patch",
"PoolTimeout",
Expand Down
39 changes: 38 additions & 1 deletion httpx/_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,13 @@
from ._urls import URL
from ._utils import get_ca_bundle_from_env

__all__ = ["Limits", "Proxy", "Timeout", "create_ssl_context"]
__all__ = ["Limits", "Proxy", "Timeout", "NetworkOptions", "create_ssl_context"]

SOCKET_OPTION = typing.Union[
typing.Tuple[int, int, int],
typing.Tuple[int, int, typing.Union[bytes, bytearray]],
typing.Tuple[int, int, None, int],
]

DEFAULT_CIPHERS = ":".join(
[
Expand Down Expand Up @@ -367,6 +373,37 @@ def __repr__(self) -> str:
return f"Proxy({url_str}{auth_str}{headers_str})"


class NetworkOptions:
def __init__(
self,
connection_retries: int = 0,
local_address: typing.Optional[str] = None,
socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None,
uds: typing.Optional[str] = None,
) -> None:
self.connection_retries = connection_retries
self.local_address = local_address
self.socket_options = socket_options
self.uds = uds

def __repr__(self) -> str:
defaults = {
"connection_retries": 0,
"local_address": None,
"socket_options": None,
"uds": None,
}
params = ", ".join(
[
f"{attr}={getattr(self, attr)!r}"
for attr, default in defaults.items()
if getattr(self, attr) != default
]
)
return f"NetworkOptions({params})"


DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0)
DEFAULT_LIMITS = Limits(max_connections=100, max_keepalive_connections=20)
DEFAULT_NETWORK_OPTIONS = NetworkOptions(connection_retries=0)
DEFAULT_MAX_REDIRECTS = 20
55 changes: 32 additions & 23 deletions httpx/_transports/default.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,14 @@

import httpcore

from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context
from .._config import (
DEFAULT_LIMITS,
DEFAULT_NETWORK_OPTIONS,
Limits,
NetworkOptions,
Proxy,
create_ssl_context,
)
from .._exceptions import (
ConnectError,
ConnectTimeout,
Expand All @@ -57,14 +64,16 @@
T = typing.TypeVar("T", bound="HTTPTransport")
A = typing.TypeVar("A", bound="AsyncHTTPTransport")


__all__ = ["AsyncHTTPTransport", "HTTPTransport"]


SOCKET_OPTION = typing.Union[
typing.Tuple[int, int, int],
typing.Tuple[int, int, typing.Union[bytes, bytearray]],
typing.Tuple[int, int, None, int],
]

__all__ = ["AsyncHTTPTransport", "HTTPTransport"]


@contextlib.contextmanager
def map_httpcore_exceptions() -> typing.Iterator[None]:
Expand Down Expand Up @@ -130,11 +139,8 @@ def __init__(
http2: bool = False,
limits: Limits = DEFAULT_LIMITS,
trust_env: bool = True,
proxy: ProxyTypes | None = None,
uds: str | None = None,
local_address: str | None = None,
retries: int = 0,
socket_options: typing.Iterable[SOCKET_OPTION] | None = None,
proxy: typing.Optional[ProxyTypes] = None,
network_options: NetworkOptions = DEFAULT_NETWORK_OPTIONS,
) -> None:
ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env)
proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy
Expand All @@ -147,10 +153,10 @@ def __init__(
keepalive_expiry=limits.keepalive_expiry,
http1=http1,
http2=http2,
uds=uds,
local_address=local_address,
retries=retries,
socket_options=socket_options,
uds=network_options.uds,
local_address=network_options.local_address,
retries=network_options.connection_retries,
socket_options=network_options.socket_options,
)
elif proxy.url.scheme in ("http", "https"):
self._pool = httpcore.HTTPProxy(
Expand All @@ -169,7 +175,10 @@ def __init__(
keepalive_expiry=limits.keepalive_expiry,
http1=http1,
http2=http2,
socket_options=socket_options,
uds=network_options.uds,
local_address=network_options.local_address,
retries=network_options.connection_retries,
socket_options=network_options.socket_options,
)
elif proxy.url.scheme == "socks5":
try:
Expand Down Expand Up @@ -271,11 +280,8 @@ def __init__(
http2: bool = False,
limits: Limits = DEFAULT_LIMITS,
trust_env: bool = True,
proxy: ProxyTypes | None = None,
uds: str | None = None,
local_address: str | None = None,
retries: int = 0,
socket_options: typing.Iterable[SOCKET_OPTION] | None = None,
proxy: typing.Optional[ProxyTypes] = None,
network_options: NetworkOptions = DEFAULT_NETWORK_OPTIONS,
) -> None:
ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env)
proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy
Expand All @@ -288,10 +294,10 @@ def __init__(
keepalive_expiry=limits.keepalive_expiry,
http1=http1,
http2=http2,
uds=uds,
local_address=local_address,
retries=retries,
socket_options=socket_options,
uds=network_options.uds,
local_address=network_options.local_address,
retries=network_options.connection_retries,
socket_options=network_options.socket_options,
)
elif proxy.url.scheme in ("http", "https"):
self._pool = httpcore.AsyncHTTPProxy(
Expand All @@ -310,7 +316,10 @@ def __init__(
keepalive_expiry=limits.keepalive_expiry,
http1=http1,
http2=http2,
socket_options=socket_options,
uds=network_options.uds,
local_address=network_options.local_address,
retries=network_options.connection_retries,
socket_options=network_options.socket_options,
)
elif proxy.url.scheme == "socks5":
try:
Expand Down
15 changes: 15 additions & 0 deletions tests/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,3 +221,18 @@ def test_proxy_with_auth_from_url():
def test_invalid_proxy_scheme():
with pytest.raises(ValueError):
httpx.Proxy("invalid://example.com")


def test_network_options():
network_options = httpx.NetworkOptions()
assert repr(network_options) == "NetworkOptions()"

network_options = httpx.NetworkOptions(connection_retries=1)
assert repr(network_options) == "NetworkOptions(connection_retries=1)"

network_options = httpx.NetworkOptions(
connection_retries=1, local_address="0.0.0.0"
)
assert repr(network_options) == (
"NetworkOptions(connection_retries=1, local_address='0.0.0.0')"
)