From 2ef5f0cd5970f57244caeff26811c98a8010440b Mon Sep 17 00:00:00 2001 From: JordonPhillips Date: Wed, 19 Mar 2025 18:18:28 +0100 Subject: [PATCH 1/4] Apply ruff check fixes --- .../software/amazon/smithy/python/codegen/PythonFormatter.java | 1 + .../smithy/python/codegen/generators/SetupGenerator.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonFormatter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonFormatter.java index 317360249..fa9b934c1 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonFormatter.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonFormatter.java @@ -72,6 +72,7 @@ private void format(FileManifest fileManifest) { return; } LOGGER.info("Running code formatter on generated code"); + CodegenUtils.runCommand("python3 -m ruff check --fix", fileManifest.getBaseDir()); CodegenUtils.runCommand("python3 -m ruff format", fileManifest.getBaseDir()); } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java index 74fbffadc..d8f0b969b 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java @@ -170,6 +170,9 @@ private static void writePyproject( [tool.ruff] target-version = "py312" + [tool.ruff.lint] + ignore = ["F841"] + [tool.pytest.ini_options] python_classes = ["!Test"] asyncio_mode = "auto" From 9b5322a53fd91067aac9d99a871a46520eb38688 Mon Sep 17 00:00:00 2001 From: JordonPhillips Date: Wed, 19 Mar 2025 16:29:00 +0100 Subject: [PATCH 2/4] Decouple endpoint resolvers from http --- .../src/smithy_aws_core/__init__.py | 7 ++ .../endpoints/standard_regional.py | 63 +++---------- .../unit/endpoints/test_standard_regional.py | 39 ++++---- .../src/smithy_core/aio/endpoints.py | 21 +++++ .../smithy_core/aio/interfaces/__init__.py | 14 ++- .../smithy-core/src/smithy_core/endpoints.py | 89 +++++++++++++++++++ .../smithy-core/src/smithy_core/exceptions.py | 4 + .../src/smithy_core/interfaces/__init__.py | 5 +- .../tests/unit/aio/test_endpoints.py | 22 ++--- .../src/smithy_http/aio/endpoints.py | 46 ---------- .../smithy_http/aio/interfaces/__init__.py | 16 +--- .../smithy-http/src/smithy_http/endpoints.py | 42 +++++---- .../smithy-http/src/smithy_http/exceptions.py | 4 - .../src/smithy_http/interfaces/__init__.py | 7 -- 14 files changed, 203 insertions(+), 176 deletions(-) create mode 100644 packages/smithy-core/src/smithy_core/aio/endpoints.py create mode 100644 packages/smithy-core/src/smithy_core/endpoints.py rename packages/{smithy-http => smithy-core}/tests/unit/aio/test_endpoints.py (55%) delete mode 100644 packages/smithy-http/src/smithy_http/aio/endpoints.py diff --git a/packages/smithy-aws-core/src/smithy_aws_core/__init__.py b/packages/smithy-aws-core/src/smithy_aws_core/__init__.py index 315342cab..03b190ea6 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/__init__.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/__init__.py @@ -4,3 +4,10 @@ import importlib.metadata __version__: str = importlib.metadata.version("smithy-aws-core") + + +from smithy_core.types import PropertyKey + + +REGION = PropertyKey(key="region", value_type=str) +"""An AWS region.""" diff --git a/packages/smithy-aws-core/src/smithy_aws_core/endpoints/standard_regional.py b/packages/smithy-aws-core/src/smithy_aws_core/endpoints/standard_regional.py index c6b37df2d..5e5b51cd6 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/endpoints/standard_regional.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/endpoints/standard_regional.py @@ -1,70 +1,29 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from dataclasses import dataclass -from typing import Protocol, Self -from urllib.parse import urlparse +from typing import Any -import smithy_core from smithy_core import URI -from smithy_http.aio.interfaces import ( - EndpointResolver, - EndpointParameters, -) -from smithy_http.endpoints import Endpoint -from smithy_http.exceptions import EndpointResolutionError +from smithy_core.aio.interfaces import EndpointResolver +from smithy_core.endpoints import Endpoint, EndpointResolverParams, resolve_static_uri +from smithy_core.exceptions import EndpointResolutionError +from .. import REGION -class _RegionUriConfig(Protocol): - endpoint_uri: str | smithy_core.interfaces.URI | None - region: str | None - -@dataclass(kw_only=True) -class RegionalEndpointParameters(EndpointParameters[_RegionUriConfig]): - """Endpoint parameters for services with standard regional endpoints.""" - - sdk_endpoint: str | smithy_core.interfaces.URI | None - region: str | None - - @classmethod - def build(cls, config: _RegionUriConfig) -> Self: - return cls(sdk_endpoint=config.endpoint_uri, region=config.region) - - -class StandardRegionalEndpointsResolver(EndpointResolver[RegionalEndpointParameters]): +class StandardRegionalEndpointsResolver(EndpointResolver): """Resolves endpoints for services with standard regional endpoints.""" def __init__(self, endpoint_prefix: str = "bedrock-runtime"): self._endpoint_prefix = endpoint_prefix - async def resolve_endpoint(self, params: RegionalEndpointParameters) -> Endpoint: - if params.sdk_endpoint is not None: - # If it's not a string, it's already a parsed URI so just pass it along. - if not isinstance(params.sdk_endpoint, str): - return Endpoint(uri=params.sdk_endpoint) - - parsed = urlparse(params.sdk_endpoint) - - # This will end up getting wrapped in the client. - if parsed.hostname is None: - raise EndpointResolutionError( - f"Unable to parse hostname from provided URI: {params.sdk_endpoint}" - ) - - return Endpoint( - uri=URI( - host=parsed.hostname, - path=parsed.path, - scheme=parsed.scheme, - query=parsed.query, - port=parsed.port, - ) - ) + async def resolve_endpoint(self, params: EndpointResolverParams[Any]) -> Endpoint: + if (static_uri := resolve_static_uri(params)) is not None: + return Endpoint(uri=static_uri) - if params.region is not None: + if (region := params.context.get(REGION)) is not None: # TODO: use dns suffix determined from partition metadata dns_suffix = "amazonaws.com" - hostname = f"{self._endpoint_prefix}.{params.region}.{dns_suffix}" + hostname = f"{self._endpoint_prefix}.{region}.{dns_suffix}" return Endpoint(uri=URI(host=hostname)) diff --git a/packages/smithy-aws-core/tests/unit/endpoints/test_standard_regional.py b/packages/smithy-aws-core/tests/unit/endpoints/test_standard_regional.py index 0ae20ac3b..ffd4f7ccf 100644 --- a/packages/smithy-aws-core/tests/unit/endpoints/test_standard_regional.py +++ b/packages/smithy-aws-core/tests/unit/endpoints/test_standard_regional.py @@ -1,21 +1,25 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from smithy_aws_core.endpoints.standard_regional import ( - StandardRegionalEndpointsResolver, - RegionalEndpointParameters, -) - -from smithy_core import URI +from unittest.mock import Mock import pytest -from smithy_http.exceptions import EndpointResolutionError +from smithy_core import URI +from smithy_core.endpoints import STATIC_URI, EndpointResolverParams +from smithy_core.types import TypedProperties +from smithy_core.exceptions import EndpointResolutionError + +from smithy_aws_core import REGION +from smithy_aws_core.endpoints.standard_regional import ( + StandardRegionalEndpointsResolver, +) async def test_resolve_endpoint_with_valid_sdk_endpoint_string(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = RegionalEndpointParameters( - sdk_endpoint="https://example.com/path?query=123", region=None + params = Mock(spec=EndpointResolverParams) + params.context = TypedProperties( + {STATIC_URI.key: "https://example.com/path?query=123"} ) endpoint = await resolver.resolve_endpoint(params) @@ -31,7 +35,8 @@ async def test_resolve_endpoint_with_sdk_endpoint_uri(): parsed_uri = URI( host="example.com", path="/path", scheme="https", query="query=123", port=443 ) - params = RegionalEndpointParameters(sdk_endpoint=parsed_uri, region=None) + params = Mock(spec=EndpointResolverParams) + params.context = TypedProperties({STATIC_URI.key: parsed_uri}) endpoint = await resolver.resolve_endpoint(params) @@ -40,7 +45,8 @@ async def test_resolve_endpoint_with_sdk_endpoint_uri(): async def test_resolve_endpoint_with_invalid_sdk_endpoint(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = RegionalEndpointParameters(sdk_endpoint="invalid-uri", region=None) + params = Mock(spec=EndpointResolverParams) + params.context = TypedProperties({STATIC_URI.key: "invalid_uri"}) with pytest.raises(EndpointResolutionError): await resolver.resolve_endpoint(params) @@ -48,7 +54,8 @@ async def test_resolve_endpoint_with_invalid_sdk_endpoint(): async def test_resolve_endpoint_with_region(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = RegionalEndpointParameters(sdk_endpoint=None, region="us-west-2") + params = Mock(spec=EndpointResolverParams) + params.context = TypedProperties({REGION.key: "us-west-2"}) endpoint = await resolver.resolve_endpoint(params) @@ -57,7 +64,8 @@ async def test_resolve_endpoint_with_region(): async def test_resolve_endpoint_with_no_sdk_endpoint_or_region(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = RegionalEndpointParameters(sdk_endpoint=None, region=None) + params = Mock(spec=EndpointResolverParams) + params.context = TypedProperties() with pytest.raises(EndpointResolutionError): await resolver.resolve_endpoint(params) @@ -65,8 +73,9 @@ async def test_resolve_endpoint_with_no_sdk_endpoint_or_region(): async def test_resolve_endpoint_with_sdk_endpoint_and_region(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = RegionalEndpointParameters( - sdk_endpoint="https://example.com", region="us-west-2" + params = Mock(spec=EndpointResolverParams) + params.context = TypedProperties( + {STATIC_URI.key: "https://example.com", REGION.key: "us-west-2"} ) endpoint = await resolver.resolve_endpoint(params) diff --git a/packages/smithy-core/src/smithy_core/aio/endpoints.py b/packages/smithy-core/src/smithy_core/aio/endpoints.py new file mode 100644 index 000000000..a41352228 --- /dev/null +++ b/packages/smithy-core/src/smithy_core/aio/endpoints.py @@ -0,0 +1,21 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from typing import Any + +from .interfaces import EndpointResolver +from ..endpoints import EndpointResolverParams, Endpoint, resolve_static_uri +from ..exceptions import EndpointResolutionError +from ..interfaces import Endpoint as _Endpoint + + +class StaticEndpointResolver(EndpointResolver): + """A basic endpoint resolver that forwards a static URI.""" + + async def resolve_endpoint(self, params: EndpointResolverParams[Any]) -> _Endpoint: + static_uri = resolve_static_uri(params) + if static_uri is None: + raise EndpointResolutionError( + "Unable to resolve endpoint: endpoint_uri is required" + ) + + return Endpoint(uri=static_uri) diff --git a/packages/smithy-core/src/smithy_core/aio/interfaces/__init__.py b/packages/smithy-core/src/smithy_core/aio/interfaces/__init__.py index bd808e42a..690179781 100644 --- a/packages/smithy-core/src/smithy_core/aio/interfaces/__init__.py +++ b/packages/smithy-core/src/smithy_core/aio/interfaces/__init__.py @@ -1,12 +1,13 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 from collections.abc import AsyncIterable -from typing import Protocol, runtime_checkable, TYPE_CHECKING, Callable +from typing import Protocol, runtime_checkable, TYPE_CHECKING, Callable, Any from ...exceptions import UnsupportedStreamException from ...interfaces import URI, Endpoint, TypedProperties from ...interfaces import StreamingBlob as SyncStreamingBlob from ...documents import TypeRegistry +from ...endpoints import EndpointResolverParams from .eventstream import EventPublisher, EventReceiver @@ -72,6 +73,17 @@ def consume_body(self) -> bytes: ... +class EndpointResolver(Protocol): + """Resolves an operation's endpoint based given parameters.""" + + async def resolve_endpoint(self, params: EndpointResolverParams[Any]) -> Endpoint: + """Resolve an endpoint for the given operation. + + :param params: The parameters available to resolve the endpoint. + """ + ... + + class ClientTransport[I: Request, O: Response](Protocol): """Protocol-agnostic representation of a client tranport (e.g. an HTTP client).""" diff --git a/packages/smithy-core/src/smithy_core/endpoints.py b/packages/smithy-core/src/smithy_core/endpoints.py new file mode 100644 index 000000000..f54c7a2be --- /dev/null +++ b/packages/smithy-core/src/smithy_core/endpoints.py @@ -0,0 +1,89 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from typing import Any +from dataclasses import dataclass, field +from urllib.parse import urlparse + +from . import URI +from .serializers import SerializeableShape +from .schemas import APIOperation +from .interfaces import TypedProperties as _TypedProperties +from .interfaces import Endpoint as _Endpoint +from .interfaces import URI as _URI +from .types import TypedProperties, PropertyKey +from .exceptions import EndpointResolutionError + + +STATIC_URI: PropertyKey[str | _URI] = PropertyKey( + key="endpoint_uri", + # Python currently has problems expressing parametric types that can be + # unions, literals, or other special types in addition to a class. So + # we have to ignore the type below. PEP 747 should resolve the issue. + # TODO: update this when PEP 747 lands + value_type=str | _URI, # type: ignore +) +"""The property key for a statically defined URI.""" + + +@dataclass(kw_only=True) +class Endpoint(_Endpoint): + """A resolved endpoint.""" + + uri: _URI + """The endpoint URI.""" + + properties: _TypedProperties = field(default_factory=TypedProperties) + """Properties required to interact with the endpoint. + + For example, in some AWS use cases this might contain HTTP headers to add to each + request. + """ + + +@dataclass(kw_only=True) +class EndpointResolverParams[I: SerializeableShape]: + """Parameters passed into an Endpoint Resolver's resolve_endpoint method.""" + + operation: APIOperation[I, Any] + """The operation to resolve an endpoint for.""" + + input: I + """The input to the operation.""" + + context: _TypedProperties + """The context of the operation invocation.""" + + +def resolve_static_uri( + properties: _TypedProperties | EndpointResolverParams[Any], +) -> _URI | None: + """Attempt to resolve a static URI from the endpoint resolver params. + + :param properties: A TypedProperties bag or EndpointResolverParams to search. + """ + properties = ( + properties.context + if isinstance(properties, EndpointResolverParams) + else properties + ) + static_uri = properties.get(STATIC_URI) + if static_uri is None: + return None + + # If it's not a string, it's already a parsed URI so just pass it along. + if not isinstance(static_uri, str): + return static_uri + + parsed = urlparse(static_uri) + if parsed.hostname is None: + raise EndpointResolutionError( + f"Unable to parse hostname from provided URI: {static_uri}" + ) + + return URI( + host=parsed.hostname, + path=parsed.path, + scheme=parsed.scheme, + query=parsed.query, + port=parsed.port, + ) diff --git a/packages/smithy-core/src/smithy_core/exceptions.py b/packages/smithy-core/src/smithy_core/exceptions.py index 104390567..0c07e30d3 100644 --- a/packages/smithy-core/src/smithy_core/exceptions.py +++ b/packages/smithy-core/src/smithy_core/exceptions.py @@ -33,3 +33,7 @@ class AsyncBodyException(SmithyException): class UnsupportedStreamException(SmithyException): """Indicates that a serializer or deserializer's stream method was called, but data streams are not supported.""" + + +class EndpointResolutionError(SmithyException): + """Exception type for all exceptions raised by endpoint resolution.""" diff --git a/packages/smithy-core/src/smithy_core/interfaces/__init__.py b/packages/smithy-core/src/smithy_core/interfaces/__init__.py index 1dc427fae..7d91713fe 100644 --- a/packages/smithy-core/src/smithy_core/interfaces/__init__.py +++ b/packages/smithy-core/src/smithy_core/interfaces/__init__.py @@ -95,15 +95,13 @@ def is_streaming_blob(obj: Any) -> TypeGuard[StreamingBlob]: return isinstance(obj, bytes | bytearray) or is_bytes_reader(obj) -# TODO: update HTTP package and existing endpoint implementations to use this. class Endpoint(Protocol): """A resolved endpoint.""" uri: URI """The endpoint URI.""" - # TODO: replace this with a typed context bag - properties: dict[str, Any] + properties: "TypedProperties" """Properties required to interact with the endpoint. For example, in some AWS use cases this might contain HTTP headers to add to each @@ -140,6 +138,7 @@ class PropertyKey[T](Protocol): key: str """The string key used to access the value.""" + # TODO: update this when PEP 747 lands to allow for unions and literals value_type: type[T] """The type of the associated value in the properties bag.""" diff --git a/packages/smithy-http/tests/unit/aio/test_endpoints.py b/packages/smithy-core/tests/unit/aio/test_endpoints.py similarity index 55% rename from packages/smithy-http/tests/unit/aio/test_endpoints.py rename to packages/smithy-core/tests/unit/aio/test_endpoints.py index e81ad0901..88021b947 100644 --- a/packages/smithy-http/tests/unit/aio/test_endpoints.py +++ b/packages/smithy-core/tests/unit/aio/test_endpoints.py @@ -1,15 +1,16 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from unittest.mock import Mock +from smithy_core.types import TypedProperties +from smithy_core.endpoints import EndpointResolverParams, STATIC_URI +from smithy_core.aio.endpoints import StaticEndpointResolver from smithy_core import URI -from smithy_http import Fields -from smithy_http.aio.endpoints import StaticEndpointResolver -from smithy_http.endpoints import StaticEndpointParams - async def test_endpoint_provider_with_uri_string() -> None: - params = StaticEndpointParams( - uri="https://foo.example.com:8080/spam?foo=bar&foo=baz" + params = Mock(spec=EndpointResolverParams) + params.context = TypedProperties( + {STATIC_URI.key: "https://foo.example.com:8080/spam?foo=bar&foo=baz"} ) expected = URI( host="foo.example.com", @@ -21,7 +22,6 @@ async def test_endpoint_provider_with_uri_string() -> None: resolver = StaticEndpointResolver() result = await resolver.resolve_endpoint(params=params) assert result.uri == expected - assert result.headers == Fields([]) async def test_endpoint_provider_with_uri_object() -> None: @@ -32,8 +32,8 @@ async def test_endpoint_provider_with_uri_object() -> None: query="foo=bar&foo=baz", port=8080, ) - params = StaticEndpointParams(uri=expected) + params = Mock(spec=EndpointResolverParams) + params.context = TypedProperties({STATIC_URI.key: expected}) resolver = StaticEndpointResolver() result = await resolver.resolve_endpoint(params=params) assert result.uri == expected - assert result.headers == Fields([]) diff --git a/packages/smithy-http/src/smithy_http/aio/endpoints.py b/packages/smithy-http/src/smithy_http/aio/endpoints.py deleted file mode 100644 index 47ebbbae2..000000000 --- a/packages/smithy-http/src/smithy_http/aio/endpoints.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -from urllib.parse import urlparse - -from smithy_core import URI -from .interfaces import EndpointResolver - -from .. import interfaces as http_interfaces -from ..endpoints import Endpoint, StaticEndpointParams -from ..exceptions import EndpointResolutionError - - -class StaticEndpointResolver(EndpointResolver[StaticEndpointParams]): - """A basic endpoint resolver that forwards a static URI.""" - - async def resolve_endpoint( - self, params: StaticEndpointParams - ) -> http_interfaces.Endpoint: - if params.uri is None: - raise EndpointResolutionError( - "Unable to resolve endpoint: endpoint_uri is required" - ) - - # If it's not a string, it's already a parsed URI so just pass it along. - if not isinstance(params.uri, str): - return Endpoint(uri=params.uri) - - # Does crt have implementations of these parsing methods? Using the standard - # library is probably fine. - parsed = urlparse(params.uri) - - # This will end up getting wrapped in the client. - if parsed.hostname is None: - raise EndpointResolutionError( - f"Unable to parse hostname from provided URI: {params.uri}" - ) - - return Endpoint( - uri=URI( - host=parsed.hostname, - path=parsed.path, - scheme=parsed.scheme, - query=parsed.query, - port=parsed.port, - ) - ) diff --git a/packages/smithy-http/src/smithy_http/aio/interfaces/__init__.py b/packages/smithy-http/src/smithy_http/aio/interfaces/__init__.py index d3a5772e2..b71a058e7 100644 --- a/packages/smithy-http/src/smithy_http/aio/interfaces/__init__.py +++ b/packages/smithy-http/src/smithy_http/aio/interfaces/__init__.py @@ -1,31 +1,17 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Protocol, Self +from typing import Protocol from smithy_core.aio.interfaces import Request, Response, ClientTransport from smithy_core.aio.utils import read_streaming_blob, read_streaming_blob_async from ...interfaces import ( - Endpoint, Fields, HTTPClientConfiguration, HTTPRequestConfiguration, ) -class EndpointParameters[C](Protocol): - @classmethod - def build(cls, config: C) -> Self: - raise NotImplementedError() - - -class EndpointResolver[T](Protocol): - """Resolves an operation's endpoint based given parameters.""" - - async def resolve_endpoint(self, params: T) -> Endpoint: - raise NotImplementedError() - - class HTTPRequest(Request, Protocol): """HTTP primitive for an Exchange to construct a version agnostic HTTP message. diff --git a/packages/smithy-http/src/smithy_http/endpoints.py b/packages/smithy-http/src/smithy_http/endpoints.py index 8f2945744..076760724 100644 --- a/packages/smithy-http/src/smithy_http/endpoints.py +++ b/packages/smithy-http/src/smithy_http/endpoints.py @@ -1,33 +1,31 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from dataclasses import dataclass, field -from typing import Protocol, Self +from dataclasses import dataclass from smithy_core.interfaces import URI +from smithy_core.interfaces import TypedProperties as _TypedProperties +from smithy_core.endpoints import Endpoint +from smithy_core.types import PropertyKey, TypedProperties from . import Fields, interfaces -from .aio.interfaces import EndpointParameters -@dataclass -class Endpoint(interfaces.Endpoint): - uri: URI - headers: interfaces.Fields = field(default_factory=Fields) +HEADERS = PropertyKey(key="headers", value_type=interfaces.Fields) +"""An Endpoint property indicating the given fields MUST be added to the request.""" -class _UriConfig(Protocol): - endpoint_uri: str | URI | None +@dataclass(init=False, kw_only=True) +class HTTPEndpoint(Endpoint): + """A resolved endpoint with optional HTTP headers.""" - -@dataclass -class StaticEndpointParams(EndpointParameters[_UriConfig]): - """Static endpoint params. - - :param uri: A static URI to route requests to. - """ - - uri: str | URI | None - - @classmethod - def build(cls, config: _UriConfig) -> Self: - return cls(uri=config.endpoint_uri) + def __init__( + self, + *, + uri: URI, + properties: _TypedProperties | None = None, + headers: interfaces.Fields | None = None, + ) -> None: + self.uri = uri + self.properties = properties if properties is not None else TypedProperties() + headers = headers if headers is not None else Fields() + self.properties[HEADERS] = headers diff --git a/packages/smithy-http/src/smithy_http/exceptions.py b/packages/smithy-http/src/smithy_http/exceptions.py index 2b27c2942..08ced3c39 100644 --- a/packages/smithy-http/src/smithy_http/exceptions.py +++ b/packages/smithy-http/src/smithy_http/exceptions.py @@ -5,7 +5,3 @@ class SmithyHTTPException(SmithyException): """Base exception type for all exceptions raised in HTTP clients.""" - - -class EndpointResolutionError(SmithyException): - """Exception type for all exceptions raised by endpoint resolution.""" diff --git a/packages/smithy-http/src/smithy_http/interfaces/__init__.py b/packages/smithy-http/src/smithy_http/interfaces/__init__.py index 2d1445199..f524ceed4 100644 --- a/packages/smithy-http/src/smithy_http/interfaces/__init__.py +++ b/packages/smithy-http/src/smithy_http/interfaces/__init__.py @@ -5,8 +5,6 @@ from enum import Enum from typing import Protocol -from smithy_core.interfaces import URI - class FieldPosition(Enum): """The type of a field. @@ -122,11 +120,6 @@ def extend(self, other: "Fields") -> None: QueryParamsList = list[tuple[str, str]] -class Endpoint(Protocol): - uri: URI - headers: Fields - - @dataclass(kw_only=True) class HTTPClientConfiguration: """Client-level HTTP configuration. From 72bb79d3be4a3ef7814e1630644abd19216d90f2 Mon Sep 17 00:00:00 2001 From: JordonPhillips Date: Wed, 19 Mar 2025 17:48:14 +0100 Subject: [PATCH 3/4] Update endpoint resolver generation --- .../python/codegen/test/AwsCodegenTest.java | 41 +++++++ .../it/resources/META-INF/smithy/main.smithy | 70 +++++++++++ .../src/it/resources/META-INF/smithy/manifest | 1 + ...sStandardRegionalEndpointsIntegration.java | 60 +-------- .../python/codegen/ClientGenerator.java | 116 ++++++++++-------- .../smithy/python/codegen/CodegenUtils.java | 28 ----- .../codegen/DirectedPythonClientCodegen.java | 2 - .../codegen/generators/ConfigGenerator.java | 38 +++--- .../generators/EndpointsGenerator.java | 59 --------- .../sections/EndpointParametersSection.java | 14 --- 10 files changed, 199 insertions(+), 230 deletions(-) create mode 100644 codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java create mode 100644 codegen/aws/core/src/it/resources/META-INF/smithy/main.smithy create mode 100644 codegen/aws/core/src/it/resources/META-INF/smithy/manifest delete mode 100644 codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EndpointsGenerator.java delete mode 100644 codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/EndpointParametersSection.java diff --git a/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java new file mode 100644 index 000000000..5bfbdba22 --- /dev/null +++ b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java @@ -0,0 +1,41 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.test; + +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.smithy.build.FileManifest; +import software.amazon.smithy.build.PluginContext; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.node.ObjectNode; +import software.amazon.smithy.python.codegen.PythonClientCodegenPlugin; + +/** + * Simple test that executes the Python client codegen plugin for an AWS-like service. + */ +public class AwsCodegenTest { + + @Test + public void testCodegen(@TempDir Path tempDir) { + PythonClientCodegenPlugin plugin = new PythonClientCodegenPlugin(); + Model model = Model.assembler(AwsCodegenTest.class.getClassLoader()) + .discoverModels(AwsCodegenTest.class.getClassLoader()) + .assemble() + .unwrap(); + PluginContext context = PluginContext.builder() + .fileManifest(FileManifest.create(tempDir)) + .settings( + ObjectNode.builder() + .withMember("service", "example.aws#RestJsonService") + .withMember("module", "restjson") + .withMember("moduleVersion", "0.0.1") + .build()) + .model(model) + .build(); + plugin.execute(context); + } + +} diff --git a/codegen/aws/core/src/it/resources/META-INF/smithy/main.smithy b/codegen/aws/core/src/it/resources/META-INF/smithy/main.smithy new file mode 100644 index 000000000..59368f205 --- /dev/null +++ b/codegen/aws/core/src/it/resources/META-INF/smithy/main.smithy @@ -0,0 +1,70 @@ +$version: "2.0" + +namespace example.aws + +use aws.protocols#restJson1 +use aws.api#service + +/// A test service that renders a restJson1 service with AWS traits +@restJson1( + http: ["h2", "http/1.1"] + eventStreamHttp: ["h2"] +) +@service( + sdkId: "REST JSON", + endpointPrefix: "rest-json-1" +) +@title("AWS REST JSON Service") +@paginated(inputToken: "nextToken", outputToken: "nextToken", pageSize: "pageSize") +@httpApiKeyAuth(name: "weather-auth", in: "header") +service RestJsonService { + version: "2006-03-01" + operations: [ + BasicOperation + ] +} + +@http(code: 200, method: "POST", uri: "/basic-operation") +operation BasicOperation { + input := { + message: String + } + output := { + message: String + } +} + +@http(code: 200, method: "POST", uri: "/input-stream") +operation InputStream { + input := { + stream: EventStream + } +} + +@http(code: 200, method: "POST", uri: "/output-stream") +operation OutputStream { + output := { + stream: EventStream + } +} + +@http(code: 200, method: "POST", uri: "/duplex-stream") +operation DuplexStream { + input := { + stream: EventStream + } + output := { + stream: EventStream + } +} + + +@streaming +union EventStream { + message: MessageEvent +} + +structure MessageEvent { + message: String +} + diff --git a/codegen/aws/core/src/it/resources/META-INF/smithy/manifest b/codegen/aws/core/src/it/resources/META-INF/smithy/manifest new file mode 100644 index 000000000..4ca2fadcf --- /dev/null +++ b/codegen/aws/core/src/it/resources/META-INF/smithy/manifest @@ -0,0 +1 @@ +main.smithy diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsStandardRegionalEndpointsIntegration.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsStandardRegionalEndpointsIntegration.java index 886abcd76..c290468b4 100644 --- a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsStandardRegionalEndpointsIntegration.java +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsStandardRegionalEndpointsIntegration.java @@ -8,12 +8,9 @@ import java.util.List; import software.amazon.smithy.aws.traits.ServiceTrait; -import software.amazon.smithy.python.codegen.CodegenUtils; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.integrations.PythonIntegration; import software.amazon.smithy.python.codegen.integrations.RuntimeClientPlugin; -import software.amazon.smithy.python.codegen.sections.EndpointParametersSection; -import software.amazon.smithy.python.codegen.sections.EndpointResolverSection; import software.amazon.smithy.python.codegen.sections.InitDefaultEndpointResolverSection; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.CodeInterceptor; @@ -37,57 +34,9 @@ public List> inte GenerationContext context ) { return List.of( - new RegionalEndpointParametersInterceptor(context), - new RegionalEndpointResolverInterceptor(context), new RegionalInitEndpointResolverInterceptor(context)); } - private static final class RegionalEndpointParametersInterceptor - implements CodeInterceptor { - - private final GenerationContext context; - - public RegionalEndpointParametersInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return EndpointParametersSection.class; - } - - @Override - public void write(PythonWriter writer, String previousText, EndpointParametersSection section) { - var params = CodegenUtils.getEndpointParametersSymbol(context.settings()); - - writer.write("from smithy_aws_core.endpoints.standard_regional import RegionalEndpointParameters"); - writer.write("$L = RegionalEndpointParameters", params.getName()); - } - } - - private static final class RegionalEndpointResolverInterceptor - implements CodeInterceptor { - - private final GenerationContext context; - - public RegionalEndpointResolverInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return EndpointResolverSection.class; - } - - @Override - public void write(PythonWriter writer, String previousText, EndpointResolverSection section) { - var resolver = CodegenUtils.getEndpointResolverSymbol(context.settings()); - - writer.write("from smithy_aws_core.endpoints.standard_regional import StandardRegionalEndpointsResolver"); - writer.write("$L = StandardRegionalEndpointsResolver", resolver.getName()); - } - } - private static final class RegionalInitEndpointResolverInterceptor implements CodeInterceptor { @@ -104,16 +53,17 @@ public Class sectionType() { @Override public void write(PythonWriter writer, String previousText, InitDefaultEndpointResolverSection section) { - var resolver = CodegenUtils.getEndpointResolverSymbol(context.settings()); - String endpointPrefix = context.settings() .service(context.model()) .getTrait(ServiceTrait.class) .map(ServiceTrait::getEndpointPrefix) .orElse(context.settings().service().getName()); - writer.write("self.endpoint_resolver = endpoint_resolver or $T(endpoint_prefix=$S)", - resolver, + writer.addImport("smithy_aws_core.endpoints.standard_regional", + "StandardRegionalEndpointsResolver", + "_RegionalResolver"); + writer.write( + "self.endpoint_resolver = endpoint_resolver or _RegionalResolver(endpoint_prefix=$S)", endpointPrefix); } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 4cca19070..22d651654 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -154,6 +154,7 @@ private void generateOperationExecutor(PythonWriter writer) { writer.addImport("smithy_core.types", "TypedProperties"); writer.addImport("smithy_core.serializers", "SerializeableShape"); writer.addImport("smithy_core.deserializers", "DeserializeableShape"); + writer.addImport("smithy_core.schemas", "APIOperation"); writer.indent(); writer.write(""" @@ -216,11 +217,11 @@ def _classify_error( serialize: Callable[[Input, $4T], Awaitable[$2T]], deserialize: Callable[[$3T, $4T], Awaitable[Output]], config: $4T, - operation_name: str, + operation: APIOperation[Input, Output], ) -> Any: request_future = Future[RequestContext[Any, $2T]]() awaitable_output = asyncio.create_task(self._execute_operation( - input, plugins, serialize, deserialize, config, operation_name, + input, plugins, serialize, deserialize, config, operation, request_future=request_future )) request_context = await request_future @@ -237,12 +238,12 @@ def _classify_error( serialize: Callable[[Input, $4T], Awaitable[$2T]], deserialize: Callable[[$3T, $4T], Awaitable[Output]], config: $4T, - operation_name: str, + operation: APIOperation[Input, Output], event_deserializer: Callable[[ShapeDeserializer], Any], ) -> Any: response_future = Future[$3T]() output = await self._execute_operation( - input, plugins, serialize, deserialize, config, operation_name, + input, plugins, serialize, deserialize, config, operation, response_future=response_future ) transport_response = await response_future @@ -259,20 +260,20 @@ def _classify_error( serialize: Callable[[Input, $4T], Awaitable[$2T]], deserialize: Callable[[$3T, $4T], Awaitable[Output]], config: $4T, - operation_name: str, + operation: APIOperation[Input, Output], event_deserializer: Callable[[ShapeDeserializer], Any], ) -> Any: request_future = Future[RequestContext[Any, $2T]]() response_future = Future[$3T]() awaitable_output = asyncio.create_task(self._execute_operation( - input, plugins, serialize, deserialize, config, operation_name, + input, plugins, serialize, deserialize, config, operation, request_future=request_future, response_future=response_future )) request_context = await request_future ${5C|} output_future = asyncio.create_task(self._wrap_duplex_output( - response_future, awaitable_output, config, operation_name, + response_future, awaitable_output, config, operation, event_deserializer )) return DuplexEventStream[Any, Any, Any]( @@ -280,12 +281,12 @@ def _classify_error( output_future=output_future, ) - async def _wrap_duplex_output( + async def _wrap_duplex_output[Input: SerializeableShape, Output: DeserializeableShape]( self, response_future: Future[$3T], awaitable_output: Future[Any], config: $4T, - operation_name: str, + operation: APIOperation[Input, Output], event_deserializer: Callable[[ShapeDeserializer], Any], ) -> tuple[Any, EventReceiver[Any]]: transport_response = await response_future @@ -310,13 +311,13 @@ async def _wrap_duplex_output( serialize: Callable[[Input, $5T], Awaitable[$2T]], deserialize: Callable[[$3T, $5T], Awaitable[Output]], config: $5T, - operation_name: str, + operation: APIOperation[Input, Output], request_future: Future[RequestContext[Any, $2T]] | None = None, response_future: Future[$3T] | None = None, ) -> Output: try: return await self._handle_execution( - input, plugins, serialize, deserialize, config, operation_name, + input, plugins, serialize, deserialize, config, operation, request_future, response_future, ) except Exception as e: @@ -338,16 +339,17 @@ async def _wrap_duplex_output( serialize: Callable[[Input, $5T], Awaitable[$2T]], deserialize: Callable[[$3T, $5T], Awaitable[Output]], config: $5T, - operation_name: str, + operation: APIOperation[Input, Output], request_future: Future[RequestContext[Any, $2T]] | None, response_future: Future[$3T] | None, ) -> Output: + operation_name = operation.schema.id.name logger.debug('Making request for operation "%s" with parameters: %s', operation_name, input) config = deepcopy(config) for plugin in plugins: plugin(config) - input_context = InputContext(request=input, properties=TypedProperties()) + input_context = InputContext(request=input, properties=TypedProperties({"config": config})) transport_request: $2T | None = None output_context: OutputContext[Input, Output, $2T | None, $3T | None] | None = None @@ -399,7 +401,7 @@ async def _wrap_duplex_output( interceptor_chain, request_context, config, - operation_name, + operation, request_future, ) @@ -454,7 +456,7 @@ await sleep(retry_token.retry_delay) interceptor: Interceptor[Input, Output, $2T, $3T], context: RequestContext[Input, $2T], config: $5T, - operation_name: str, + operation: APIOperation[Input, Output], request_future: Future[RequestContext[Input, $2T]] | None, ) -> OutputContext[Input, Output, $2T, $3T | None]: transport_response: $3T | None = None @@ -476,7 +478,7 @@ await sleep(retry_token.retry_delay) writer.write(""" # Step 7b: Invoke service_auth_scheme_resolver.resolve_auth_scheme auth_parameters: $1T = $1T( - operation=operation_name, + operation=operation.schema.id.name, ${2C|} ) @@ -522,37 +524,44 @@ await sleep(retry_token.retry_delay) writer.popState(); writer.pushState(new ResolveEndpointSection()); + writer.addDependency(SmithyPythonDependency.SMITHY_CORE); + writer.addDependency(SmithyPythonDependency.SMITHY_HTTP); + writer.addImport("smithy_core", "URI"); + writer.addImport("smithy_core.endpoints", "EndpointResolverParams"); + writer.write(""" + # Step 7f: Invoke endpoint_resolver.resolve_endpoint + context.properties["endpoint_uri"] = config.endpoint_uri + endpoint_resolver_parameters = EndpointResolverParams( + operation=operation, + input=context.request, + context=context.properties + ) + logger.debug("Calling endpoint resolver with parameters: %s", endpoint_resolver_parameters) + endpoint = await config.endpoint_resolver.resolve_endpoint( + endpoint_resolver_parameters + ) + logger.debug("Endpoint resolver result: %s", endpoint) + if not endpoint.uri.path: + path = "" + elif endpoint.uri.path.endswith("/"): + path = endpoint.uri.path[:-1] + else: + path = endpoint.uri.path + if context.transport_request.destination.path: + path += context.transport_request.destination.path + context.transport_request.destination = URI( + scheme=endpoint.uri.scheme, + host=context.transport_request.destination.host + endpoint.uri.host, + path=path, + port=endpoint.uri.port, + query=context.transport_request.destination.query, + ) + """); if (context.applicationProtocol().isHttpProtocol()) { - writer.addDependency(SmithyPythonDependency.SMITHY_CORE); - writer.addDependency(SmithyPythonDependency.SMITHY_HTTP); - writer.addImport("smithy_core", "URI"); writer.write(""" - # Step 7f: Invoke endpoint_resolver.resolve_endpoint - endpoint_resolver_parameters = $1T.build(config=config) - logger.debug("Calling endpoint resolver with parameters: %s", endpoint_resolver_parameters) - endpoint = await config.endpoint_resolver.resolve_endpoint( - endpoint_resolver_parameters - ) - logger.debug("Endpoint resolver result: %s", endpoint) - if not endpoint.uri.path: - path = "" - elif endpoint.uri.path.endswith("/"): - path = endpoint.uri.path[:-1] - else: - path = endpoint.uri.path - if context.transport_request.destination.path: - path += context.transport_request.destination.path - context.transport_request.destination = URI( - scheme=endpoint.uri.scheme, - host=context.transport_request.destination.host + endpoint.uri.host, - path=path, - port=endpoint.uri.port, - query=context.transport_request.destination.query, - ) - context.transport_request.fields.extend(endpoint.headers) - - """, - CodegenUtils.getEndpointParametersSymbol(context.settings())); + if (headers := endpoint.properties.get("headers")) is not None: + context.transport_request.fields.extend(headers) + """); } writer.popState(); @@ -788,7 +797,8 @@ private void writeDefaultPlugins(PythonWriter writer, Collection writeSharedOperationInit(w, operation, input))); @@ -972,7 +984,7 @@ raise NotImplementedError() serialize=${serSymbol:T}, deserialize=${deserSymbol:T}, config=self._config, - operation_name=${operationName:S}, + operation=${operation:T}, event_deserializer=$T().deserialize, ) # type: ignore ${/hasProtocol} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java index 6660e1851..4cba69d64 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java @@ -139,34 +139,6 @@ public static Symbol getUnknownApiError(PythonSettings settings) { .build(); } - /** - * Gets the symbol for the endpoint parameters object. - * - * @param settings The client settings, used to account for module configuration. - * @return Returns the symbol for endpoint parameters. - */ - public static Symbol getEndpointParametersSymbol(PythonSettings settings) { - return Symbol.builder() - .name("EndpointParameters") - .namespace(String.format("%s.endpoints", settings.moduleName()), ".") - .definitionFile(String.format("./%s/endpoints.py", settings.moduleName())) - .build(); - } - - /** - * Gets the symbol for the endpoint resolver object. - * - * @param settings The client settings, used to account for module configuration. - * @return Returns the symbol for endpoint resolver. - */ - public static Symbol getEndpointResolverSymbol(PythonSettings settings) { - return Symbol.builder() - .name("EndpointResolver") - .namespace(String.format("%s.endpoints", settings.moduleName()), ".") - .definitionFile(String.format("./%s/endpoints.py", settings.moduleName())) - .build(); - } - /** * Gets the symbol for the http auth params. * diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonClientCodegen.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonClientCodegen.java index 54a2531f0..5c02c146f 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonClientCodegen.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonClientCodegen.java @@ -30,7 +30,6 @@ import software.amazon.smithy.model.shapes.ServiceShape; import software.amazon.smithy.model.shapes.ShapeId; import software.amazon.smithy.python.codegen.generators.ConfigGenerator; -import software.amazon.smithy.python.codegen.generators.EndpointsGenerator; import software.amazon.smithy.python.codegen.generators.EnumGenerator; import software.amazon.smithy.python.codegen.generators.InitGenerator; import software.amazon.smithy.python.codegen.generators.IntEnumGenerator; @@ -114,7 +113,6 @@ public void customizeBeforeShapeGeneration(CustomizeDirective BASE_PROPERTIES = Arrays.asList( + private static final List BASE_PROPERTIES = List.of( ConfigProperty.builder() .name("interceptors") .type(Symbol.builder() @@ -67,19 +66,6 @@ public final class ConfigGenerator implements Runnable { writer.addImport("smithy_core.retries", "SimpleRetryStrategy"); writer.write("self.retry_strategy = retry_strategy or SimpleRetryStrategy()"); }) - .build()); - - // This list contains any properties that must be added to any http-based - // service client, except for the http client itself. - private static final List HTTP_PROPERTIES = Arrays.asList( - ConfigProperty.builder() - .name("http_request_config") - .type(Symbol.builder() - .name("HTTPRequestConfiguration") - .namespace("smithy_http.interfaces", ".") - .addDependency(SmithyPythonDependency.SMITHY_HTTP) - .build()) - .documentation("Configuration for individual HTTP requests.") .build(), ConfigProperty.builder() .name("endpoint_uri") @@ -94,6 +80,19 @@ public final class ConfigGenerator implements Runnable { .documentation("A static URI to route requests to.") .build()); + // This list contains any properties that must be added to any http-based + // service client, except for the http client itself. + private static final List HTTP_PROPERTIES = List.of( + ConfigProperty.builder() + .name("http_request_config") + .type(Symbol.builder() + .name("HTTPRequestConfiguration") + .namespace("smithy_http.interfaces", ".") + .addDependency(SmithyPythonDependency.SMITHY_HTTP) + .build()) + .documentation("Configuration for individual HTTP requests.") + .build()); + private final PythonSettings settings; private final GenerationContext context; @@ -132,21 +131,20 @@ private static List getHttpProperties(GenerationContext context) } properties.add(clientBuilder.build()); - var endpointResolver = CodegenUtils.getEndpointResolverSymbol(context.settings()); properties.add(ConfigProperty.builder() .name("endpoint_resolver") .type(Symbol.builder() - .name("_EndpointResolver[EndpointParameters]") - .addReference(CodegenUtils.getEndpointParametersSymbol(context.settings())) + .name("_EndpointResolver") .build()) .documentation(""" The endpoint resolver used to resolve the final endpoint per-operation based on the \ configuration.""") .nullable(false) .initialize(writer -> { - writer.addImport("smithy_http.aio.interfaces", "EndpointResolver", "_EndpointResolver"); + writer.addImport("smithy_core.aio.interfaces", "EndpointResolver", "_EndpointResolver"); writer.pushState(new InitDefaultEndpointResolverSection()); - writer.write("self.endpoint_resolver = endpoint_resolver or $T()", endpointResolver); + writer.addImport("smithy_core.aio.endpoints", "StaticEndpointResolver"); + writer.write("self.endpoint_resolver = endpoint_resolver or StaticEndpointResolver()"); writer.popState(); }) .build()); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EndpointsGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EndpointsGenerator.java deleted file mode 100644 index a913ec800..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EndpointsGenerator.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.generators; - -import software.amazon.smithy.codegen.core.Symbol; -import software.amazon.smithy.python.codegen.CodegenUtils; -import software.amazon.smithy.python.codegen.GenerationContext; -import software.amazon.smithy.python.codegen.PythonSettings; -import software.amazon.smithy.python.codegen.SmithyPythonDependency; -import software.amazon.smithy.python.codegen.sections.EndpointParametersSection; -import software.amazon.smithy.python.codegen.sections.EndpointResolverSection; -import software.amazon.smithy.python.codegen.writer.PythonWriter; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * This class is responsible for generating the endpoint resolver and its parameters. - */ -@SmithyInternalApi -public final class EndpointsGenerator implements Runnable { - - private final PythonSettings settings; - private final GenerationContext context; - - public EndpointsGenerator(GenerationContext context, PythonSettings settings) { - this.context = context; - this.settings = settings; - } - - @Override - public void run() { - var params = CodegenUtils.getEndpointParametersSymbol(settings); - context.writerDelegator().useFileWriter(params.getDefinitionFile(), params.getNamespace(), writer -> { - generateEndpointParameters(writer, params); - }); - - var resolver = CodegenUtils.getEndpointResolverSymbol(settings); - context.writerDelegator().useFileWriter(resolver.getDefinitionFile(), resolver.getNamespace(), writer -> { - generateEndpointResolver(writer, resolver); - }); - } - - private void generateEndpointParameters(PythonWriter writer, Symbol params) { - writer.pushState(new EndpointParametersSection()); - writer.addDependency(SmithyPythonDependency.SMITHY_HTTP); - writer.write("from smithy_http.endpoints import StaticEndpointParams"); - writer.write("$L = StaticEndpointParams", params.getName()); - writer.popState(); - } - - private void generateEndpointResolver(PythonWriter writer, Symbol resolver) { - writer.pushState(new EndpointResolverSection()); - writer.addDependency(SmithyPythonDependency.SMITHY_HTTP); - writer.write("from smithy_http.aio.endpoints import StaticEndpointResolver"); - writer.write("$L = StaticEndpointResolver", resolver.getName()); - writer.popState(); - } -} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/EndpointParametersSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/EndpointParametersSection.java deleted file mode 100644 index b70a0b111..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/EndpointParametersSection.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.sections; - -import software.amazon.smithy.utils.CodeSection; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * A section that controls generating the EndpointParameters class. - */ -@SmithyInternalApi -public record EndpointParametersSection() implements CodeSection {} From 3a67247a7be8e1bd21715c833fa154bd00b377c4 Mon Sep 17 00:00:00 2001 From: JordonPhillips Date: Thu, 20 Mar 2025 16:00:00 +0100 Subject: [PATCH 4/4] Pull resolver params from config --- .../python/codegen/ClientGenerator.java | 1 - .../src/smithy_aws_core/__init__.py | 7 --- .../src/smithy_aws_core/endpoints/__init__.py | 13 +++++ .../endpoints/standard_regional.py | 7 +-- .../unit/endpoints/test_standard_regional.py | 49 ++++++++++++------- .../smithy-core/src/smithy_core/endpoints.py | 30 ++++++------ .../src/smithy_core/interfaces/__init__.py | 1 - .../tests/unit/aio/test_endpoints.py | 27 +++++++--- 8 files changed, 85 insertions(+), 50 deletions(-) diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 22d651654..041678e6f 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -530,7 +530,6 @@ await sleep(retry_token.retry_delay) writer.addImport("smithy_core.endpoints", "EndpointResolverParams"); writer.write(""" # Step 7f: Invoke endpoint_resolver.resolve_endpoint - context.properties["endpoint_uri"] = config.endpoint_uri endpoint_resolver_parameters = EndpointResolverParams( operation=operation, input=context.request, diff --git a/packages/smithy-aws-core/src/smithy_aws_core/__init__.py b/packages/smithy-aws-core/src/smithy_aws_core/__init__.py index 03b190ea6..315342cab 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/__init__.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/__init__.py @@ -4,10 +4,3 @@ import importlib.metadata __version__: str = importlib.metadata.version("smithy-aws-core") - - -from smithy_core.types import PropertyKey - - -REGION = PropertyKey(key="region", value_type=str) -"""An AWS region.""" diff --git a/packages/smithy-aws-core/src/smithy_aws_core/endpoints/__init__.py b/packages/smithy-aws-core/src/smithy_aws_core/endpoints/__init__.py index 33cbe867a..3f6aa0edf 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/endpoints/__init__.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/endpoints/__init__.py @@ -1,2 +1,15 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +from smithy_core.endpoints import StaticEndpointConfig +from smithy_core.types import PropertyKey + + +class RegionalEndpointConfig(StaticEndpointConfig): + """Endpoint config for services with standard regional endpoints.""" + + region: str | None + """The AWS region to address the request to.""" + + +REGIONAL_ENDPOINT_CONFIG = PropertyKey(key="config", value_type=RegionalEndpointConfig) +"""Endpoint config for services with standard regional endpoints.""" diff --git a/packages/smithy-aws-core/src/smithy_aws_core/endpoints/standard_regional.py b/packages/smithy-aws-core/src/smithy_aws_core/endpoints/standard_regional.py index 5e5b51cd6..4610b750a 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/endpoints/standard_regional.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/endpoints/standard_regional.py @@ -7,7 +7,7 @@ from smithy_core.endpoints import Endpoint, EndpointResolverParams, resolve_static_uri from smithy_core.exceptions import EndpointResolutionError -from .. import REGION +from . import REGIONAL_ENDPOINT_CONFIG class StandardRegionalEndpointsResolver(EndpointResolver): @@ -20,10 +20,11 @@ async def resolve_endpoint(self, params: EndpointResolverParams[Any]) -> Endpoin if (static_uri := resolve_static_uri(params)) is not None: return Endpoint(uri=static_uri) - if (region := params.context.get(REGION)) is not None: + region_config = params.context.get(REGIONAL_ENDPOINT_CONFIG) + if region_config is not None and region_config.region is not None: # TODO: use dns suffix determined from partition metadata dns_suffix = "amazonaws.com" - hostname = f"{self._endpoint_prefix}.{region}.{dns_suffix}" + hostname = f"{self._endpoint_prefix}.{region_config.region}.{dns_suffix}" return Endpoint(uri=URI(host=hostname)) diff --git a/packages/smithy-aws-core/tests/unit/endpoints/test_standard_regional.py b/packages/smithy-aws-core/tests/unit/endpoints/test_standard_regional.py index ffd4f7ccf..15d6aeedb 100644 --- a/packages/smithy-aws-core/tests/unit/endpoints/test_standard_regional.py +++ b/packages/smithy-aws-core/tests/unit/endpoints/test_standard_regional.py @@ -1,26 +1,46 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass +from typing import Any from unittest.mock import Mock import pytest from smithy_core import URI -from smithy_core.endpoints import STATIC_URI, EndpointResolverParams +from smithy_core.endpoints import EndpointResolverParams from smithy_core.types import TypedProperties from smithy_core.exceptions import EndpointResolutionError -from smithy_aws_core import REGION +from smithy_aws_core.endpoints import REGIONAL_ENDPOINT_CONFIG from smithy_aws_core.endpoints.standard_regional import ( StandardRegionalEndpointsResolver, ) +@dataclass +class EndpointConfig: + endpoint_uri: str | URI | None = None + region: str | None = None + + @classmethod + def params( + cls, endpoint_uri: str | URI | None = None, region: str | None = None + ) -> EndpointResolverParams[Any]: + properties = TypedProperties( + { + REGIONAL_ENDPOINT_CONFIG.key: cls( + endpoint_uri=endpoint_uri, region=region + ) + } + ) + params = Mock(spec=EndpointResolverParams) + params.context = properties + return params + + async def test_resolve_endpoint_with_valid_sdk_endpoint_string(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = Mock(spec=EndpointResolverParams) - params.context = TypedProperties( - {STATIC_URI.key: "https://example.com/path?query=123"} - ) + params = EndpointConfig.params("https://example.com/path?query=123") endpoint = await resolver.resolve_endpoint(params) @@ -35,8 +55,7 @@ async def test_resolve_endpoint_with_sdk_endpoint_uri(): parsed_uri = URI( host="example.com", path="/path", scheme="https", query="query=123", port=443 ) - params = Mock(spec=EndpointResolverParams) - params.context = TypedProperties({STATIC_URI.key: parsed_uri}) + params = EndpointConfig.params(parsed_uri) endpoint = await resolver.resolve_endpoint(params) @@ -45,8 +64,7 @@ async def test_resolve_endpoint_with_sdk_endpoint_uri(): async def test_resolve_endpoint_with_invalid_sdk_endpoint(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = Mock(spec=EndpointResolverParams) - params.context = TypedProperties({STATIC_URI.key: "invalid_uri"}) + params = EndpointConfig.params("invalid_uri") with pytest.raises(EndpointResolutionError): await resolver.resolve_endpoint(params) @@ -54,8 +72,7 @@ async def test_resolve_endpoint_with_invalid_sdk_endpoint(): async def test_resolve_endpoint_with_region(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = Mock(spec=EndpointResolverParams) - params.context = TypedProperties({REGION.key: "us-west-2"}) + params = EndpointConfig.params(region="us-west-2") endpoint = await resolver.resolve_endpoint(params) @@ -64,8 +81,7 @@ async def test_resolve_endpoint_with_region(): async def test_resolve_endpoint_with_no_sdk_endpoint_or_region(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = Mock(spec=EndpointResolverParams) - params.context = TypedProperties() + params = EndpointConfig.params() with pytest.raises(EndpointResolutionError): await resolver.resolve_endpoint(params) @@ -73,9 +89,8 @@ async def test_resolve_endpoint_with_no_sdk_endpoint_or_region(): async def test_resolve_endpoint_with_sdk_endpoint_and_region(): resolver = StandardRegionalEndpointsResolver(endpoint_prefix="service") - params = Mock(spec=EndpointResolverParams) - params.context = TypedProperties( - {STATIC_URI.key: "https://example.com", REGION.key: "us-west-2"} + params = EndpointConfig.params( + endpoint_uri="https://example.com", region="us-west-2" ) endpoint = await resolver.resolve_endpoint(params) diff --git a/packages/smithy-core/src/smithy_core/endpoints.py b/packages/smithy-core/src/smithy_core/endpoints.py index f54c7a2be..9f07eae96 100644 --- a/packages/smithy-core/src/smithy_core/endpoints.py +++ b/packages/smithy-core/src/smithy_core/endpoints.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Any +from typing import Any, Protocol from dataclasses import dataclass, field from urllib.parse import urlparse @@ -14,17 +14,6 @@ from .exceptions import EndpointResolutionError -STATIC_URI: PropertyKey[str | _URI] = PropertyKey( - key="endpoint_uri", - # Python currently has problems expressing parametric types that can be - # unions, literals, or other special types in addition to a class. So - # we have to ignore the type below. PEP 747 should resolve the issue. - # TODO: update this when PEP 747 lands - value_type=str | _URI, # type: ignore -) -"""The property key for a statically defined URI.""" - - @dataclass(kw_only=True) class Endpoint(_Endpoint): """A resolved endpoint.""" @@ -54,6 +43,17 @@ class EndpointResolverParams[I: SerializeableShape]: """The context of the operation invocation.""" +class StaticEndpointConfig(Protocol): + """A config that has a static endpoint.""" + + endpoint_uri: str | URI | None + """A static endpoint to use for the request.""" + + +STATIC_ENDPOINT_CONFIG = PropertyKey(key="config", value_type=StaticEndpointConfig) +"""Property containing a config that has a static endpoint.""" + + def resolve_static_uri( properties: _TypedProperties | EndpointResolverParams[Any], ) -> _URI | None: @@ -66,10 +66,12 @@ def resolve_static_uri( if isinstance(properties, EndpointResolverParams) else properties ) - static_uri = properties.get(STATIC_URI) - if static_uri is None: + static_uri_config = properties.get(STATIC_ENDPOINT_CONFIG) + if static_uri_config is None or static_uri_config.endpoint_uri is None: return None + static_uri = static_uri_config.endpoint_uri + # If it's not a string, it's already a parsed URI so just pass it along. if not isinstance(static_uri, str): return static_uri diff --git a/packages/smithy-core/src/smithy_core/interfaces/__init__.py b/packages/smithy-core/src/smithy_core/interfaces/__init__.py index 7d91713fe..a38110a0a 100644 --- a/packages/smithy-core/src/smithy_core/interfaces/__init__.py +++ b/packages/smithy-core/src/smithy_core/interfaces/__init__.py @@ -138,7 +138,6 @@ class PropertyKey[T](Protocol): key: str """The string key used to access the value.""" - # TODO: update this when PEP 747 lands to allow for unions and literals value_type: type[T] """The type of the associated value in the properties bag.""" diff --git a/packages/smithy-core/tests/unit/aio/test_endpoints.py b/packages/smithy-core/tests/unit/aio/test_endpoints.py index 88021b947..5346c4be2 100644 --- a/packages/smithy-core/tests/unit/aio/test_endpoints.py +++ b/packages/smithy-core/tests/unit/aio/test_endpoints.py @@ -1,17 +1,31 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass +from typing import Any from unittest.mock import Mock from smithy_core.types import TypedProperties -from smithy_core.endpoints import EndpointResolverParams, STATIC_URI +from smithy_core.endpoints import EndpointResolverParams, STATIC_ENDPOINT_CONFIG from smithy_core.aio.endpoints import StaticEndpointResolver from smithy_core import URI +@dataclass +class EndpointConfig: + endpoint_uri: str | URI | None = None + + @classmethod + def params( + cls, endpoint_uri: str | URI | None = None + ) -> EndpointResolverParams[Any]: + params = Mock(spec=EndpointResolverParams) + params.context = TypedProperties( + {STATIC_ENDPOINT_CONFIG.key: cls(endpoint_uri)} + ) + return params + + async def test_endpoint_provider_with_uri_string() -> None: - params = Mock(spec=EndpointResolverParams) - params.context = TypedProperties( - {STATIC_URI.key: "https://foo.example.com:8080/spam?foo=bar&foo=baz"} - ) + params = EndpointConfig.params("https://foo.example.com:8080/spam?foo=bar&foo=baz") expected = URI( host="foo.example.com", path="/spam", @@ -32,8 +46,7 @@ async def test_endpoint_provider_with_uri_object() -> None: query="foo=bar&foo=baz", port=8080, ) - params = Mock(spec=EndpointResolverParams) - params.context = TypedProperties({STATIC_URI.key: expected}) + params = EndpointConfig.params(expected) resolver = StaticEndpointResolver() result = await resolver.resolve_endpoint(params=params) assert result.uri == expected