From 83b5e4bf130d204fbb25b26a341c62aee4fc2d0f Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 12 Jan 2024 12:14:15 +0000 Subject: [PATCH 1/4] Add NetworkOptions --- httpx/__init__.py | 3 ++- httpx/_config.py | 38 +++++++++++++++++++++++++++ httpx/_transports/default.py | 51 ++++++++++++++++++------------------ tests/test_config.py | 15 +++++++++++ 4 files changed, 81 insertions(+), 26 deletions(-) diff --git a/httpx/__init__.py b/httpx/__init__.py index f61112f8b2..1665b35a1f 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -2,7 +2,7 @@ from ._api import delete, get, head, options, patch, post, put, request, stream from ._auth import Auth, BasicAuth, DigestAuth, NetRCAuth from ._client import USE_CLIENT_DEFAULT, AsyncClient, Client -from ._config import Limits, Proxy, Timeout, create_ssl_context +from ._config import Limits, NetworkOptions, Proxy, Timeout, create_ssl_context from ._content import ByteStream from ._exceptions import ( CloseError, @@ -96,6 +96,7 @@ def main() -> None: # type: ignore "MockTransport", "NetRCAuth", "NetworkError", + "NetworkOptions", "options", "patch", "PoolTimeout", diff --git a/httpx/_config.py b/httpx/_config.py index 0cfd552e49..69c3c6ffa8 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -12,6 +12,13 @@ from ._urls import URL from ._utils import get_ca_bundle_from_env + +SOCKET_OPTION = typing.Union[ + typing.Tuple[int, int, int], + typing.Tuple[int, int, typing.Union[bytes, bytearray]], + typing.Tuple[int, int, None, int], +] + DEFAULT_CIPHERS = ":".join( [ "ECDHE+AESGCM", @@ -363,6 +370,37 @@ def __repr__(self) -> str: return f"Proxy({url_str}{auth_str}{headers_str})" +class NetworkOptions: + def __init__( + self, + connection_retries: int = 0, + local_address: typing.Optional[str] = None, + socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None, + uds: typing.Optional[str] = None, + ) -> None: + self.connection_retries = connection_retries + self.local_address = local_address + self.socket_options = socket_options + self.uds = uds + + def __repr__(self) -> str: + defaults = { + "connection_retries": 0, + "local_address": None, + "socket_options": None, + "uds": None, + } + params = ", ".join( + [ + f"{attr}={getattr(self, attr)!r}" + for attr, default in defaults.items() + if getattr(self, attr) != default + ] + ) + return f"NetworkOptions({params})" + + DEFAULT_TIMEOUT_CONFIG = Timeout(timeout=5.0) DEFAULT_LIMITS = Limits(max_connections=100, max_keepalive_connections=20) +DEFAULT_NETWORK_OPTIONS = NetworkOptions(connection_retries=0) DEFAULT_MAX_REDIRECTS = 20 diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index 14a087389a..7802026940 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -29,7 +29,14 @@ import httpcore -from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context +from .._config import ( + DEFAULT_LIMITS, + DEFAULT_NETWORK_OPTIONS, + Proxy, + Limits, + NetworkOptions, + create_ssl_context, +) from .._exceptions import ( ConnectError, ConnectTimeout, @@ -54,12 +61,6 @@ T = typing.TypeVar("T", bound="HTTPTransport") A = typing.TypeVar("A", bound="AsyncHTTPTransport") -SOCKET_OPTION = typing.Union[ - typing.Tuple[int, int, int], - typing.Tuple[int, int, typing.Union[bytes, bytearray]], - typing.Tuple[int, int, None, int], -] - @contextlib.contextmanager def map_httpcore_exceptions() -> typing.Iterator[None]: @@ -126,10 +127,7 @@ def __init__( limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, proxy: typing.Optional[ProxyTypes] = None, - uds: typing.Optional[str] = None, - local_address: typing.Optional[str] = None, - retries: int = 0, - socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None, + network_options: NetworkOptions = DEFAULT_NETWORK_OPTIONS, ) -> None: ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy @@ -142,10 +140,10 @@ def __init__( keepalive_expiry=limits.keepalive_expiry, http1=http1, http2=http2, - uds=uds, - local_address=local_address, - retries=retries, - socket_options=socket_options, + uds=network_options.uds, + local_address=network_options.local_address, + retries=network_options.connection_retries, + socket_options=network_options.socket_options, ) elif proxy.url.scheme in ("http", "https"): self._pool = httpcore.HTTPProxy( @@ -164,7 +162,10 @@ def __init__( keepalive_expiry=limits.keepalive_expiry, http1=http1, http2=http2, - socket_options=socket_options, + uds=network_options.uds, + local_address=network_options.local_address, + retries=network_options.connection_retries, + socket_options=network_options.socket_options, ) elif proxy.url.scheme == "socks5": try: @@ -267,10 +268,7 @@ def __init__( limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, proxy: typing.Optional[ProxyTypes] = None, - uds: typing.Optional[str] = None, - local_address: typing.Optional[str] = None, - retries: int = 0, - socket_options: typing.Optional[typing.Iterable[SOCKET_OPTION]] = None, + network_options: NetworkOptions = DEFAULT_NETWORK_OPTIONS, ) -> None: ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy @@ -283,10 +281,10 @@ def __init__( keepalive_expiry=limits.keepalive_expiry, http1=http1, http2=http2, - uds=uds, - local_address=local_address, - retries=retries, - socket_options=socket_options, + uds=network_options.uds, + local_address=network_options.local_address, + retries=network_options.connection_retries, + socket_options=network_options.socket_options, ) elif proxy.url.scheme in ("http", "https"): self._pool = httpcore.AsyncHTTPProxy( @@ -304,7 +302,10 @@ def __init__( keepalive_expiry=limits.keepalive_expiry, http1=http1, http2=http2, - socket_options=socket_options, + uds=network_options.uds, + local_address=network_options.local_address, + retries=network_options.connection_retries, + socket_options=network_options.socket_options, ) elif proxy.url.scheme == "socks5": try: diff --git a/tests/test_config.py b/tests/test_config.py index 6f6ee4f575..cef0aa37c2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -221,3 +221,18 @@ def test_proxy_with_auth_from_url(): def test_invalid_proxy_scheme(): with pytest.raises(ValueError): httpx.Proxy("invalid://example.com") + + +def test_network_options(): + network_options = httpx.NetworkOptions() + assert repr(network_options) == "NetworkOptions()" + + network_options = httpx.NetworkOptions(connection_retries=1) + assert repr(network_options) == "NetworkOptions(connection_retries=1)" + + network_options = httpx.NetworkOptions( + connection_retries=1, local_address="0.0.0.0" + ) + assert repr(network_options) == ( + "NetworkOptions(connection_retries=1, local_address='0.0.0.0')" + ) From 913ea35324c99c2052331008ea8a4b8037e5b4cb Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 12 Jan 2024 12:18:49 +0000 Subject: [PATCH 2/4] Linting --- httpx/_config.py | 1 - httpx/_transports/default.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/httpx/_config.py b/httpx/_config.py index 69c3c6ffa8..1af8a4565a 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -12,7 +12,6 @@ from ._urls import URL from ._utils import get_ca_bundle_from_env - SOCKET_OPTION = typing.Union[ typing.Tuple[int, int, int], typing.Tuple[int, int, typing.Union[bytes, bytearray]], diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index 7802026940..0829b5704c 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -32,9 +32,9 @@ from .._config import ( DEFAULT_LIMITS, DEFAULT_NETWORK_OPTIONS, - Proxy, Limits, NetworkOptions, + Proxy, create_ssl_context, ) from .._exceptions import ( From 6ac2101706c7380d37b286258ad1ef5663eee29d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 16 Jan 2024 10:53:35 +0000 Subject: [PATCH 3/4] Documentation --- docs/advanced/network-options.md | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 docs/advanced/network-options.md diff --git a/docs/advanced/network-options.md b/docs/advanced/network-options.md new file mode 100644 index 0000000000..08329359d1 --- /dev/null +++ b/docs/advanced/network-options.md @@ -0,0 +1,33 @@ +There are several advanced network options that are made available through the `httpx.NetworkOptions` configuration class. + +```python +# Configure an HTTPTransport with some specific network options. +network_options = httpx.NetworkOptions( + connection_retries=1, + local_address="0.0.0.0", +) +transport = httpx.HTTPTransport(network_options=network_options) + +# Instantiate a client with the configured transport. +client = httpx.Client(transport=transport) +``` + +## Configuration + +The options available on this class are... + +### `connection_retries` + +Configure a number of retries that may be attempted when initially establishing a TCP connection. Defaults to `0`. + +### `local_address` + +Configure the local address that the socket should be bound too. The most common usage is for enforcing binding to either IPv4 `local_address="0.0.0.0"` or IPv6 `local_address="::"`. + +### `socket_options` + +*TODO: Example* + +### `uds` + +Connect to a Unix Domain Socket, rather than over the network. Should be a string providing the path to the UDS. \ No newline at end of file From acd83dac70eb9df52518add421498959a5f157e7 Mon Sep 17 00:00:00 2001 From: Kar Petrosyan <92274156+karpetrosyan@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:25:24 +0400 Subject: [PATCH 4/4] Add documentation for socket_options --- docs/advanced/network-options.md | 13 ++++++++++++- httpx/_config.py | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/advanced/network-options.md b/docs/advanced/network-options.md index 08329359d1..ea444d4206 100644 --- a/docs/advanced/network-options.md +++ b/docs/advanced/network-options.md @@ -26,7 +26,18 @@ Configure the local address that the socket should be bound too. The most common ### `socket_options` -*TODO: Example* +Configure the list of socket options to be applied to the underlying sockets used for network connections. +For example, you can use it to explicitly specify which network interface should be used for the connection in this manner: + +```python +import httpx + +socket_options = [(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, b"ETH999")] + +network_options = httpx.NetworkOptions( + socket_options=socket_options +) +``` ### `uds` diff --git a/httpx/_config.py b/httpx/_config.py index 736cfdf38d..168b4438fe 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -14,7 +14,7 @@ from ._urls import URL from ._utils import get_ca_bundle_from_env -__all__ = ["Limits", "Proxy", "Timeout", "create_ssl_context"] +__all__ = ["Limits", "Proxy", "Timeout", "NetworkOptions", "create_ssl_context"] SOCKET_OPTION = typing.Union[ typing.Tuple[int, int, int],