diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0bb6185d8..36eb193e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ To send us a pull request, please: 1. Fork the repository. 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 3. Ensure local tests pass (`make test-py` and `make test-protocols`). -4. Run `make lint-py` if you've changed any python sources. +4. Run `make lint-py` and `make check-py` if you've changed any python sources. 4. Commit to your fork using clear commit messages. 5. Send us a pull request, answering any default questions in the pull request interface. 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsPythonDependency.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsPythonDependency.java new file mode 100644 index 000000000..b8d0dc2dc --- /dev/null +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsPythonDependency.java @@ -0,0 +1,26 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.aws.codegen; + +import software.amazon.smithy.python.codegen.PythonDependency; +import software.amazon.smithy.utils.SmithyUnstableApi; + +/** + * AWS Dependencies used in the smithy python generator. + */ +@SmithyUnstableApi +public class AwsPythonDependency { + /** + * The core aws smithy runtime python package. + * + *

While in development this will use the develop branch. + */ + public static final PythonDependency SMITHY_AWS_CORE = new PythonDependency( + "smithy_aws_core", + // You'll need to locally install this before we publish + "==0.0.1", + PythonDependency.Type.DEPENDENCY, + false); +} diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsUserAgentIntegration.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsUserAgentIntegration.java new file mode 100644 index 000000000..f51d0ec6e --- /dev/null +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsUserAgentIntegration.java @@ -0,0 +1,114 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.aws.codegen; + +import java.util.List; +import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.integrations.PythonIntegration; +import software.amazon.smithy.python.codegen.integrations.RuntimeClientPlugin; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Adds a runtime plugin to set user agent. + */ +@SmithyInternalApi +public class AwsUserAgentIntegration implements PythonIntegration { + + public static final String USER_AGENT_PLUGIN = """ + def aws_user_agent_plugin(config: $1T): + config.interceptors.append( + $2T( + ua_suffix=config.user_agent_extra, + ua_app_id=config.sdk_ua_app_id, + sdk_version=$3T, + service_id='$4L' + ) + ) + """; + + @Override + public List getClientPlugins(GenerationContext context) { + if (context.applicationProtocol().isHttpProtocol()) { + final ConfigProperty userAgentExtra = ConfigProperty.builder() + .name("user_agent_extra") + .documentation("Additional suffix to be added to the User-Agent header.") + .type(Symbol.builder().name("str").build()) + .nullable(true) + .build(); + + final ConfigProperty uaAppId = ConfigProperty.builder() + .name("sdk_ua_app_id") + .documentation( + "A unique and opaque application ID that is appended to the User-Agent header.") + .type(Symbol.builder().name("str").build()) + .nullable(true) + .build(); + + final String user_agent_plugin_file = "user_agent"; + + final String moduleName = context.settings().moduleName(); + final SymbolReference userAgentPlugin = SymbolReference.builder() + .symbol(Symbol.builder() + .namespace(String.format("%s.%s", + moduleName, + user_agent_plugin_file.replace('/', '.')), ".") + .definitionFile(String + .format("./%s/%s.py", moduleName, user_agent_plugin_file)) + .name("aws_user_agent_plugin") + .build()) + .build(); + final SymbolReference userAgentInterceptor = SymbolReference.builder() + .symbol(Symbol.builder() + .namespace("smithy_aws_core.interceptors.user_agent", ".") + .name("UserAgentInterceptor") + .build()) + .build(); + final SymbolReference versionSymbol = SymbolReference.builder() + .symbol(Symbol.builder() + .namespace(moduleName, ".") + .name("__version__") + .build()) + .build(); + + final String serviceId = context.settings() + .service(context.model()) + .getTrait(ServiceTrait.class) + .map(ServiceTrait::getSdkId) + .orElse(context.settings().service().getName()) + .replace(' ', '_'); + + return List.of( + RuntimeClientPlugin.builder() + .addConfigProperty(userAgentExtra) + .addConfigProperty(uaAppId) + .pythonPlugin(userAgentPlugin) + .writeAdditionalFiles((c) -> { + String filename = "%s/%s.py".formatted(moduleName, user_agent_plugin_file); + c.writerDelegator() + .useFileWriter( + filename, + moduleName + ".", + writer -> { + writer.write(USER_AGENT_PLUGIN, + CodegenUtils.getConfigSymbol(c.settings()), + userAgentInterceptor, + versionSymbol, + serviceId); + + }); + return List.of(filename); + }) + .build()); + } else { + return List.of(); + } + } + +} diff --git a/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index 5155ed74a..0375294ba 100644 --- a/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -5,3 +5,4 @@ software.amazon.smithy.python.aws.codegen.AwsAuthIntegration software.amazon.smithy.python.aws.codegen.AwsProtocolsIntegration +software.amazon.smithy.python.aws.codegen.AwsUserAgentIntegration diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 5aa42d257..f6639a94d 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -76,7 +76,7 @@ private void generateService(PythonWriter writer) { var defaultPlugins = new LinkedHashSet(); for (PythonIntegration integration : context.integrations()) { - for (RuntimeClientPlugin runtimeClientPlugin : integration.getClientPlugins()) { + for (RuntimeClientPlugin runtimeClientPlugin : integration.getClientPlugins(context)) { if (runtimeClientPlugin.matchesService(context.model(), service)) { runtimeClientPlugin.getPythonPlugin().ifPresent(defaultPlugins::add); } @@ -678,7 +678,7 @@ private boolean hasEventStream() { private void initializeHttpAuthParameters(PythonWriter writer) { var derived = new LinkedHashSet(); for (PythonIntegration integration : context.integrations()) { - for (RuntimeClientPlugin plugin : integration.getClientPlugins()) { + for (RuntimeClientPlugin plugin : integration.getClientPlugins(context)) { if (plugin.matchesService(context.model(), service) && plugin.getAuthScheme().isPresent() && plugin.getAuthScheme().get().getApplicationProtocol().isHttpProtocol()) { @@ -767,7 +767,7 @@ private void writeSharedOperationInit(PythonWriter writer, OperationShape operat var defaultPlugins = new LinkedHashSet(); for (PythonIntegration integration : context.integrations()) { - for (RuntimeClientPlugin runtimeClientPlugin : integration.getClientPlugins()) { + for (RuntimeClientPlugin runtimeClientPlugin : integration.getClientPlugins(context)) { if (runtimeClientPlugin.matchesOperation(context.model(), service, operation)) { runtimeClientPlugin.getPythonPlugin().ifPresent(defaultPlugins::add); } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java index a52cb082e..5a161912c 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/DirectedPythonCodegen.java @@ -50,6 +50,7 @@ import software.amazon.smithy.python.codegen.generators.StructureGenerator; import software.amazon.smithy.python.codegen.generators.UnionGenerator; import software.amazon.smithy.python.codegen.integrations.PythonIntegration; +import software.amazon.smithy.python.codegen.integrations.RuntimeClientPlugin; import software.amazon.smithy.python.codegen.writer.PythonDelegator; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyUnstableApi; @@ -273,9 +274,39 @@ public void generateIntEnumShape(GenerateIntEnumDirective directive) { + generateServiceModuleInit(directive); + generatePluginFiles(directive); generateInits(directive); } + /** + * Writes out all extra files required by runtime plugins. + */ + private void generatePluginFiles(CustomizeDirective directive) { + GenerationContext context = directive.context(); + for (PythonIntegration integration : context.integrations()) { + for (RuntimeClientPlugin runtimeClientPlugin : integration.getClientPlugins(context)) { + if (runtimeClientPlugin.matchesService(context.model(), directive.service())) { + runtimeClientPlugin.writeAdditionalFiles(context); + } + } + } + } + + /** + * Creates top level __init__.py file. + */ + private void generateServiceModuleInit(CustomizeDirective directive) { + directive.context() + .writerDelegator() + .useFileWriter( + "%s/__init__.py".formatted(directive.context().settings().moduleName()), + writer -> { + writer + .write("__version__: str = '$L'", directive.context().settings().moduleVersion()); + }); + } + /** * Creates __init__.py files where not already present. */ diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpAuthGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpAuthGenerator.java index 5500d0849..843196d85 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpAuthGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpAuthGenerator.java @@ -43,7 +43,7 @@ public void run() { var properties = new ArrayList(); var service = context.settings().service(context.model()); for (PythonIntegration integration : context.integrations()) { - for (RuntimeClientPlugin plugin : integration.getClientPlugins()) { + for (RuntimeClientPlugin plugin : integration.getClientPlugins(context)) { if (plugin.matchesService(context.model(), service) && plugin.getAuthScheme().isPresent() && plugin.getAuthScheme().get().getApplicationProtocol().isHttpProtocol()) { diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java index fb01926d8..1d120d6d9 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java @@ -219,7 +219,7 @@ private static void writeDefaultHttpAuthSchemes(GenerationContext context, Pytho var supportedAuthSchemes = new LinkedHashMap(); var service = context.settings().service(context.model()); for (PythonIntegration integration : context.integrations()) { - for (RuntimeClientPlugin plugin : integration.getClientPlugins()) { + for (RuntimeClientPlugin plugin : integration.getClientPlugins(context)) { if (plugin.matchesService(context.model(), service) && plugin.getAuthScheme().isPresent() && plugin.getAuthScheme().get().getApplicationProtocol().isHttpProtocol()) { @@ -289,7 +289,7 @@ private void writeInterceptorsType(PythonWriter writer) { } private void generateConfig(GenerationContext context, PythonWriter writer) { - var symbol = CodegenUtils.getConfigSymbol(context.settings()); + var configSymbol = CodegenUtils.getConfigSymbol(context.settings()); // Initialize the list of config properties with our base properties. Here a new // list is constructed because that base list is immutable. @@ -312,7 +312,7 @@ private void generateConfig(GenerationContext context, PythonWriter writer) { // Add any relevant config properties from plugins. for (PythonIntegration integration : context.integrations()) { - for (RuntimeClientPlugin plugin : integration.getClientPlugins()) { + for (RuntimeClientPlugin plugin : integration.getClientPlugins(context)) { if (plugin.matchesService(model, service)) { properties.addAll(plugin.getConfigProperties()); } @@ -340,7 +340,7 @@ def __init__( \""" ${C|} """, - symbol.getName(), + configSymbol.getName(), context.settings().service().getName(), writer.consumer(w -> writePropertyDeclarations(w, finalProperties)), writer.consumer(w -> writeInitParams(w, finalProperties)), diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/HttpApiKeyAuth.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/HttpApiKeyAuth.java index f8034d0ab..f7b292d85 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/HttpApiKeyAuth.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/HttpApiKeyAuth.java @@ -24,7 +24,7 @@ public final class HttpApiKeyAuth implements PythonIntegration { private static final String OPTION_GENERATOR_NAME = "_generate_api_key_option"; @Override - public List getClientPlugins() { + public List getClientPlugins(GenerationContext context) { return List.of( RuntimeClientPlugin.builder() .servicePredicate((model, service) -> service.hasTrait(HttpApiKeyAuthTrait.class)) diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/PythonIntegration.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/PythonIntegration.java index a23cf237a..6a6f37338 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/PythonIntegration.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/PythonIntegration.java @@ -35,7 +35,7 @@ default List getProtocolGenerators() { * * @return Returns the list of RuntimePlugins to apply to the client. */ - default List getClientPlugins() { + default List getClientPlugins(GenerationContext context) { return Collections.emptyList(); } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RuntimeClientPlugin.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RuntimeClientPlugin.java index 88ac5ec4f..540cdf50b 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RuntimeClientPlugin.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RuntimeClientPlugin.java @@ -16,6 +16,7 @@ import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.ServiceShape; import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.utils.SmithyBuilder; import software.amazon.smithy.utils.SmithyUnstableApi; import software.amazon.smithy.utils.ToSmithyBuilder; @@ -38,6 +39,7 @@ public final class RuntimeClientPlugin implements ToSmithyBuilder configProperties; private final SymbolReference pythonPlugin; + private final WriteAdditionalFiles writeAdditionalFiles; private final AuthScheme authScheme; @@ -47,6 +49,7 @@ private RuntimeClientPlugin(Builder builder) { configProperties = Collections.unmodifiableList(builder.configProperties); this.pythonPlugin = builder.pythonPlugin; this.authScheme = builder.authScheme; + this.writeAdditionalFiles = builder.writeAdditionalFiles; } /** @@ -65,6 +68,20 @@ public interface OperationPredicate { boolean test(Model model, ServiceShape service, OperationShape operation); } + @FunctionalInterface + /** + * Called to write out additional files. + */ + public interface WriteAdditionalFiles { + /** + * Called to write out additional files needed by a generator. + * + * @param context GenerationContext - allows access to file manifest and symbol providers + * @return List of the relative paths of files written. + */ + List writeAdditionalFiles(GenerationContext context); + } + /** * Returns true if this plugin applies to the given service. * @@ -120,6 +137,16 @@ public Optional getAuthScheme() { return Optional.ofNullable(authScheme); } + /** + * Write additional files required by this plugin. + * + * @param context generation context + * @return relative paths of additional files written. + */ + public List writeAdditionalFiles(GenerationContext context) { + return writeAdditionalFiles.writeAdditionalFiles(context); + } + /** * @return Returns a new builder for a {@link RuntimeClientPlugin}. */ @@ -132,7 +159,8 @@ public SmithyBuilder toBuilder() { var builder = builder() .pythonPlugin(pythonPlugin) .authScheme(authScheme) - .configProperties(configProperties); + .configProperties(configProperties) + .writeAdditionalFiles(writeAdditionalFiles); if (operationPredicate == OPERATION_ALWAYS_FALSE) { builder.servicePredicate(servicePredicate); @@ -152,6 +180,7 @@ public static final class Builder implements SmithyBuilder private List configProperties = new ArrayList<>(); private SymbolReference pythonPlugin = null; private AuthScheme authScheme = null; + private WriteAdditionalFiles writeAdditionalFiles = (context) -> Collections.emptyList(); Builder() {} @@ -264,5 +293,16 @@ public Builder authScheme(AuthScheme authScheme) { this.authScheme = authScheme; return this; } + + /** + * Write additional files required by this plugin. + * + * @param writeAdditionalFiles additional files to write. + * @return Returns the builder. + */ + public Builder writeAdditionalFiles(WriteAdditionalFiles writeAdditionalFiles) { + this.writeAdditionalFiles = writeAdditionalFiles; + return this; + } } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/UserAgentIntegration.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/UserAgentIntegration.java new file mode 100644 index 000000000..5d1df673c --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/UserAgentIntegration.java @@ -0,0 +1,38 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.integrations; + +import java.util.List; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Adds a runtime plugin to set generic user agent. + */ +@SmithyInternalApi +public class UserAgentIntegration implements PythonIntegration { + @Override + public List getClientPlugins(GenerationContext context) { + if (context.applicationProtocol().isHttpProtocol()) { + var userAgentPlugin = SymbolReference + .builder() + .symbol( + Symbol.builder() + .namespace( + SmithyPythonDependency.SMITHY_HTTP.packageName() + ".plugins", + ".") + .name("user_agent_plugin") + .addDependency(SmithyPythonDependency.SMITHY_HTTP) + .build()) + .build(); + return List.of(RuntimeClientPlugin.builder().pythonPlugin(userAgentPlugin).build()); + } else { + return List.of(); + } + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/ImportDeclarations.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/ImportDeclarations.java index 799b64912..13df49e9d 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/ImportDeclarations.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/ImportDeclarations.java @@ -70,7 +70,12 @@ private String relativize(String namespace) { } } var prefix = StringUtils.repeat(".", localParts.length - commonSegments); - return prefix + namespace.split("\\.", commonSegments + 1)[commonSegments]; + String[] segments = namespace.split("\\.", commonSegments + 1); + if (commonSegments >= segments.length) { + return "."; + } else { + return prefix + segments[commonSegments]; + } } ImportDeclarations addStdlibImport(String namespace) { diff --git a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index e8dc8f92c..154d4fc1f 100644 --- a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -5,3 +5,4 @@ software.amazon.smithy.python.codegen.integrations.RestJsonIntegration software.amazon.smithy.python.codegen.integrations.HttpApiKeyAuth +software.amazon.smithy.python.codegen.integrations.UserAgentIntegration diff --git a/packages/smithy-aws-core/src/smithy_aws_core/interceptors/__init__.py b/packages/smithy-aws-core/src/smithy_aws_core/interceptors/__init__.py new file mode 100644 index 000000000..33cbe867a --- /dev/null +++ b/packages/smithy-aws-core/src/smithy_aws_core/interceptors/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/packages/smithy-aws-core/src/smithy_aws_core/interceptors/user_agent.py b/packages/smithy-aws-core/src/smithy_aws_core/interceptors/user_agent.py new file mode 100644 index 000000000..6acccdeff --- /dev/null +++ b/packages/smithy-aws-core/src/smithy_aws_core/interceptors/user_agent.py @@ -0,0 +1,72 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# pyright: reportMissingTypeStubs=false +from typing import Any + +import smithy_aws_core +import smithy_core +from smithy_core.interceptors import Interceptor, InterceptorContext +from smithy_http.user_agent import UserAgentComponent, RawStringUserAgentComponent + +_USERAGENT_SDK_NAME = "aws-sdk-python" + + +class UserAgentInterceptor(Interceptor[Any, Any, Any, Any]): + """Adds AWS fields to the UserAgent.""" + + def __init__( + self, + *, + ua_suffix: str | None, + ua_app_id: str | None, + sdk_version: str, + service_id: str, + ) -> None: + """Initialize the UserAgentInterceptor. + + :param ua_suffix: Additional suffix to be added to the UserAgent header. + :param ua_app_id: User defined and opaque application ID to be added to the + UserAgent header. + :param sdk_version: SDK version to be added to the UserAgent header. + :param service_id: ServiceId to be added to the UserAgent header. + """ + super().__init__() + self._ua_suffix = ua_suffix + self._ua_app_id = ua_app_id + self._sdk_version = sdk_version + self._service_id = service_id + + def read_after_serialization( + self, context: InterceptorContext[Any, Any, Any, Any] + ) -> None: + if "user_agent" in context.properties: + user_agent = context.properties["user_agent"] + user_agent.sdk_metadata = self._build_sdk_metadata() + user_agent.api_metadata.append( + UserAgentComponent("api", self._service_id, self._sdk_version) + ) + + if self._ua_app_id is not None: + user_agent.additional_metadata.append( + UserAgentComponent("app", self._ua_app_id) + ) + + if self._ua_suffix is not None: + user_agent.additional_metadata.append( + RawStringUserAgentComponent(self._ua_suffix) + ) + + def _build_sdk_metadata(self) -> list[UserAgentComponent]: + return [ + UserAgentComponent(_USERAGENT_SDK_NAME, smithy_aws_core.__version__), + UserAgentComponent("md", "smithy-core", smithy_core.__version__), + *self._crt_version(), + ] + + def _crt_version(self) -> list[UserAgentComponent]: + try: + import awscrt + + return [UserAgentComponent("md", "awscrt", awscrt.__version__)] + except AttributeError: + return [] diff --git a/packages/smithy-http/src/smithy_http/interceptors/__init__.py b/packages/smithy-http/src/smithy_http/interceptors/__init__.py new file mode 100644 index 000000000..33cbe867a --- /dev/null +++ b/packages/smithy-http/src/smithy_http/interceptors/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/packages/smithy-http/src/smithy_http/interceptors/user_agent.py b/packages/smithy-http/src/smithy_http/interceptors/user_agent.py new file mode 100644 index 000000000..ad58ddcaa --- /dev/null +++ b/packages/smithy-http/src/smithy_http/interceptors/user_agent.py @@ -0,0 +1,151 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import platform +from typing import Self, Any + +import smithy_core +from smithy_core.interceptors import Interceptor, InterceptorContext +from smithy_http import Field +from smithy_http.aio.interfaces import HTTPRequest +from smithy_http.user_agent import UserAgent, UserAgentComponent + + +class UserAgentInterceptor(Interceptor[Any, None, HTTPRequest, None]): + """Adds interceptors that initialize UserAgent in the context and add the user-agent + header.""" + + def read_before_execution( + self, context: InterceptorContext[Any, None, None, None] + ) -> None: + context.properties["user_agent"] = _UserAgentBuilder.from_environment().build() + + def modify_before_signing( + self, context: InterceptorContext[Any, None, HTTPRequest, None] + ) -> HTTPRequest: + user_agent = context.properties["user_agent"] + request = context.transport_request + request.fields.set_field(Field(name="User-Agent", values=[str(user_agent)])) + return context.transport_request + + +_USERAGENT_ALLOWED_OS_NAMES = ( + "windows", + "linux", + "macos", + "android", + "ios", + "watchos", + "tvos", + "other", +) +_USERAGENT_PLATFORM_NAME_MAPPINGS = {"darwin": "macos"} +_USERAGENT_SDK_NAME = "python" + + +class _UserAgentBuilder: + def __init__( + self, + *, + platform_name: str | None, + platform_version: str | None, + platform_machine: str | None, + python_version: str | None, + python_implementation: str | None, + sdk_version: str | None, + ) -> None: + self._platform_name = platform_name + self._platform_version = platform_version + self._platform_machine = platform_machine + self._python_version = python_version + self._python_implementation = python_implementation + self._sdk_version = sdk_version + + @classmethod + def from_environment(cls) -> Self: + return cls( + platform_name=platform.system(), + platform_version=platform.release(), + platform_machine=platform.machine(), + python_version=platform.python_version(), + python_implementation=platform.python_implementation(), + sdk_version=smithy_core.__version__, + ) + + def build(self) -> UserAgent: + user_agent = UserAgent() + user_agent.sdk_metadata.extend(self._build_sdk_metadata()) + user_agent.ua_metadata.append(UserAgentComponent(prefix="ua", name="2.1")) + user_agent.os_metadata.extend(self._build_os_metadata()) + user_agent.os_metadata.extend(self._build_architecture_metadata()) + user_agent.language_metadata.extend(self._build_language_metadata()) + + return user_agent + + def _build_sdk_metadata(self) -> list[UserAgentComponent]: + if self._sdk_version: + return [ + UserAgentComponent(prefix=_USERAGENT_SDK_NAME, name=self._sdk_version) + ] + return [] + + def _build_os_metadata(self) -> list[UserAgentComponent]: + """Build the OS/platform components of the User-Agent header string. + + For recognized platform names that match or map to an entry in the list + of standardized OS names, a single component with prefix "os" is + returned. Otherwise, one component "os/other" is returned and a second + with prefix "md" and the raw platform name. + + String representations of example return values: + * ``os/macos#10.13.6`` + * ``os/linux`` + * ``os/other`` + * ``os/other md/foobar#1.2.3`` + """ + if self._platform_name in (None, ""): + return [UserAgentComponent("os", "other")] + + plt_name_lower = self._platform_name.lower() + if plt_name_lower in _USERAGENT_ALLOWED_OS_NAMES: + os_family = plt_name_lower + elif plt_name_lower in _USERAGENT_PLATFORM_NAME_MAPPINGS: + os_family = _USERAGENT_PLATFORM_NAME_MAPPINGS[plt_name_lower] + else: + os_family = None + + if os_family is not None: + return [UserAgentComponent("os", os_family, self._platform_version)] + else: + return [ + UserAgentComponent("os", "other"), + UserAgentComponent("md", self._platform_name, self._platform_version), + ] + + def _build_architecture_metadata(self) -> list[UserAgentComponent]: + """Build architecture component of the User-Agent header string. + + Returns the machine type with prefix "md" and name "arch", if one is available. + Common values include "x86_64", "arm64", "i386". + """ + if self._platform_machine: + return [UserAgentComponent("md", "arch", self._platform_machine.lower())] + return [] + + def _build_language_metadata(self) -> list[UserAgentComponent]: + """Build the language components of the User-Agent header string. + + Returns the Python version in a component with prefix "lang" and name + "python". The Python implementation (e.g. CPython, PyPy) is returned as + separate metadata component with prefix "md" and name "pyimpl". + + String representation of an example return value: + ``lang/python#3.10.4 md/pyimpl#CPython`` + """ + lang_md = [ + UserAgentComponent("lang", "python", self._python_version), + ] + if self._python_implementation: + lang_md.append( + UserAgentComponent("md", "pyimpl", self._python_implementation) + ) + return lang_md diff --git a/packages/smithy-http/src/smithy_http/plugins/__init__.py b/packages/smithy-http/src/smithy_http/plugins/__init__.py new file mode 100644 index 000000000..2d2ab0574 --- /dev/null +++ b/packages/smithy-http/src/smithy_http/plugins/__init__.py @@ -0,0 +1,14 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from typing import Protocol, Any + +from smithy_core.interceptors import Interceptor +from smithy_http.interceptors.user_agent import UserAgentInterceptor + + +class _InterceptorConfig(Protocol): + interceptors: list[Interceptor[Any, Any, Any, Any]] + + +def user_agent_plugin(config: _InterceptorConfig) -> None: + config.interceptors.append(UserAgentInterceptor()) diff --git a/packages/smithy-http/src/smithy_http/user_agent.py b/packages/smithy-http/src/smithy_http/user_agent.py new file mode 100644 index 000000000..e44e3ca11 --- /dev/null +++ b/packages/smithy-http/src/smithy_http/user_agent.py @@ -0,0 +1,97 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# pyright: reportMissingTypeStubs=false,reportUnknownMemberType=false + +from dataclasses import dataclass, field +from string import ascii_letters, digits + +_USERAGENT_ALLOWED_CHARACTERS = ascii_letters + digits + "!$%&'*+-.^_`|~" + + +@dataclass(frozen=True, slots=True) +class UserAgentComponent: + """Component of a User-Agent header string in the standard format. + + Each component consists of a prefix, a name, and a value. In the string + representation these are combined in the format ``prefix/name#value``. + + This class is considered private and is subject to abrupt breaking changes. + """ + + prefix: str + name: str + value: str | None = None + + def __str__(self): + """Create string like 'prefix/name#value' from a UserAgentComponent.""" + clean_prefix = sanitize_user_agent_string_component( + self.prefix, allow_hash=True + ) + clean_name = sanitize_user_agent_string_component(self.name, allow_hash=False) + if self.value is None or self.value == "": + return f"{clean_prefix}/{clean_name}" + clean_value = sanitize_user_agent_string_component(self.value, allow_hash=True) + return f"{clean_prefix}/{clean_name}#{clean_value}" + + +@dataclass(frozen=True, slots=True) +class RawStringUserAgentComponent: + """UserAgentComponent interface wrapper around ``str``. + + Use for User-Agent header components that are not constructed from prefix+name+value + but instead are provided as strings. No sanitization is performed. + """ + + value: str + + def __str__(self) -> str: + return self.value + + +@dataclass(kw_only=True, slots=True) +class UserAgent: + sdk_metadata: list[UserAgentComponent] = field(default_factory=list) + internal_metadata: list[UserAgentComponent] = field(default_factory=list) + ua_metadata: list[UserAgentComponent] = field(default_factory=list) + api_metadata: list[UserAgentComponent] = field(default_factory=list) + os_metadata: list[UserAgentComponent] = field(default_factory=list) + language_metadata: list[UserAgentComponent] = field(default_factory=list) + env_metadata: list[UserAgentComponent] = field(default_factory=list) + config_metadata: list[UserAgentComponent] = field(default_factory=list) + feat_metadata: list[UserAgentComponent] = field(default_factory=list) + additional_metadata: list[UserAgentComponent | RawStringUserAgentComponent] = field( + default_factory=list + ) + + def __str__(self) -> str: + components = [ + *self.sdk_metadata, + *self.internal_metadata, + *self.ua_metadata, + *self.api_metadata, + *self.os_metadata, + *self.language_metadata, + *self.env_metadata, + *self.config_metadata, + *self.feat_metadata, + *self.additional_metadata, + ] + return " ".join([str(comp) for comp in components]) + + +def sanitize_user_agent_string_component(raw_str: str, allow_hash: bool = False) -> str: + """Replaces all not allowed characters in the string with a dash ("-"). + + Allowed characters are ASCII alphanumerics and ``!$%&'*+-.^_`|~``. If + ``allow_hash`` is ``True``, "#"``" is also allowed. + + :type raw_str: str + :param raw_str: The input string to be sanitized. + + :type allow_hash: bool + :param allow_hash: Whether "#" is considered an allowed character. + """ + return "".join( + c if c in _USERAGENT_ALLOWED_CHARACTERS or (allow_hash and c == "#") else "-" + for c in raw_str + ) diff --git a/packages/smithy-http/tests/unit/interceptors/test_user_agent.py b/packages/smithy-http/tests/unit/interceptors/test_user_agent.py new file mode 100644 index 000000000..6f73e887d --- /dev/null +++ b/packages/smithy-http/tests/unit/interceptors/test_user_agent.py @@ -0,0 +1,89 @@ +import platform + +import smithy_core +from smithy_http.interceptors.user_agent import _UserAgentBuilder # type: ignore + + +def test_from_environment(monkeypatch): # type: ignore + monkeypatch.setattr(platform, "system", lambda: "Linux") # type: ignore + monkeypatch.setattr(platform, "release", lambda: "5.4.228-131.415.AMZN2.X86_64") # type: ignore + monkeypatch.setattr(platform, "machine", lambda: "x86_64") # type: ignore + monkeypatch.setattr(platform, "python_version", lambda: "4.3.2") # type: ignore + monkeypatch.setattr(platform, "python_implementation", lambda: "CPython") # type: ignore + monkeypatch.setattr(smithy_core, "__version__", "1.2.3") # type: ignore + + user_agent = str(_UserAgentBuilder.from_environment().build()) + assert "python/1.2.3" in user_agent + assert "os/linux#5.4.228-131.415.AMZN2.X86_64" in user_agent + assert "md/arch#x86_64" in user_agent + assert "lang/python#4.3.2" in user_agent + assert "md/pyimpl#CPython" in user_agent + + +defaults = { + "platform_name": None, + "platform_version": None, + "platform_machine": None, + "python_version": None, + "python_implementation": None, + "sdk_version": None, +} + + +def test_build_adds_sdk_metadata(): + args = {"sdk_version": "1.2.3"} + user_agent = _UserAgentBuilder(**{**defaults, **args}).build() + assert "python/1.2.3" in str(user_agent) + + +def test_build_adds_ua_metadata(): + user_agent = _UserAgentBuilder(**defaults).build() + assert "ua/2.1" in str(user_agent) + + +def test_build_os_defaults_to_other(): + user_agent = _UserAgentBuilder(**defaults).build() + assert "os/other" in str(user_agent) + + +def test_build_os_lowercases_platform(): + args = {"platform_name": "LINUX"} + user_agent = _UserAgentBuilder(**{**defaults, **args}).build() + assert "os/linux" in str(user_agent) + + +def test_build_os_maps_platform_names(): + args = {"platform_name": "darwin"} + user_agent = _UserAgentBuilder(**{**defaults, **args}).build() + + assert "os/macos" in str(user_agent) + + +def test_build_os_includes_version(): + args = {"platform_name": "linux", "platform_version": "5.4"} + user_agent = _UserAgentBuilder(**{**defaults, **args}).build() + assert "os/linux#5.4" in str(user_agent) + + +def test_build_os_other_platform(): + args = {"platform_name": "myos", "platform_version": "0.0.1"} + user_agent = _UserAgentBuilder(**{**defaults, **args}).build() + assert "os/other md/myos#0.0.1" in str(user_agent) + + +def test_build_arch_adds_md(): + args = {"platform_machine": "x86_64"} + user_agent = _UserAgentBuilder(**{**defaults, **args}).build() + assert "md/arch#x86_64" in str(user_agent) + + +def test_build_language_version(): + args = {"python_version": "3.12"} + user_agent = _UserAgentBuilder(**{**defaults, **args}).build() + assert "lang/python#3.12" in str(user_agent) + + +def test_build_language_implementation(): + args = {"python_implementation": "CPython"} + user_agent = _UserAgentBuilder(**{**defaults, **args}).build() + assert "md/pyimpl#CPython" in str(user_agent) diff --git a/packages/smithy-http/tests/unit/test_user_agent.py b/packages/smithy-http/tests/unit/test_user_agent.py new file mode 100644 index 000000000..dcf135807 --- /dev/null +++ b/packages/smithy-http/tests/unit/test_user_agent.py @@ -0,0 +1,121 @@ +import pytest + +from smithy_http.user_agent import ( + sanitize_user_agent_string_component, + UserAgentComponent, + RawStringUserAgentComponent, + UserAgent, +) + + +@pytest.mark.parametrize( + "raw_str, allow_hash, expected_str", + [ + ("foo", False, "foo"), + ("foo", True, "foo"), + ("ExampleFramework (1.2.3)", False, "ExampleFramework--1.2.3-"), + ("foo#1.2.3", False, "foo-1.2.3"), + ("foo#1.2.3", True, "foo#1.2.3"), + ("", False, ""), + ("", True, ""), + ("", False, ""), + ("#", False, "-"), + ("#", True, "#"), + (" ", False, "-"), + (" ", False, "--"), + ("@=[]{ }/\\øß©", True, "------------"), + ( + "Java_HotSpot_(TM)_64-Bit_Server_VM/25.151-b12", + True, + "Java_HotSpot_-TM-_64-Bit_Server_VM-25.151-b12", + ), + ], +) +def test_sanitize_ua_string_component( + raw_str: str, allow_hash: bool, expected_str: str +): + actual_str = sanitize_user_agent_string_component(raw_str, allow_hash) + assert actual_str == expected_str + + +# Test cases for UserAgentComponent +def test_user_agent_component_without_value(): + component = UserAgentComponent(prefix="md", name="test") + assert str(component) == "md/test" + + +def test_user_agent_component_with_value(): + component = UserAgentComponent(prefix="md", name="test", value="123") + assert str(component) == "md/test#123" + + +def test_user_agent_component_with_empty_value(): + component = UserAgentComponent(prefix="md", name="test", value="") + assert str(component) == "md/test" + + +def test_user_agent_component_sanitization(): + component = UserAgentComponent(prefix="md@", name="test!", value="123#") + assert str(component) == "md-/test!#123#" + + +# Test cases for RawStringUserAgentComponent +def test_raw_string_user_agent_component(): + component = RawStringUserAgentComponent(value="raw/string#123") + assert str(component) == "raw/string#123" + + +# Test cases for UserAgent +def test_user_agent_with_multiple_components(): + sdk_component = UserAgentComponent(prefix="sdk", name="python", value="1.0") + os_component = UserAgentComponent(prefix="os", name="linux", value="5.4") + raw_component = RawStringUserAgentComponent(value="raw/string#123") + + user_agent = UserAgent( + sdk_metadata=[sdk_component], + os_metadata=[os_component], + additional_metadata=[raw_component], + ) + + expected_output = "sdk/python#1.0 os/linux#5.4 raw/string#123" + assert str(user_agent) == expected_output + + +def test_user_agent_with_empty_metadata(): + user_agent = UserAgent() + assert str(user_agent) == "" + + +def test_user_agent_with_all_metadata_types(): + sdk_component = UserAgentComponent(prefix="sdk", name="python", value="1.0") + internal_component = UserAgentComponent(prefix="md", name="internal") + ua_component = UserAgentComponent(prefix="ua", name="2.0") + api_component = UserAgentComponent(prefix="api", name="operation", value="1.1") + os_component = UserAgentComponent(prefix="os", name="linux", value="5.4") + language_component = UserAgentComponent(prefix="lang", name="python", value="3.12") + env_component = UserAgentComponent(prefix="exec-env", name="prod") + config_component = UserAgentComponent( + prefix="cfg", name="retry-mode", value="standard" + ) + feat_component = UserAgentComponent(prefix="ft", name="paginator") + raw_component = RawStringUserAgentComponent(value="raw/string#123") + + user_agent = UserAgent( + sdk_metadata=[sdk_component], + internal_metadata=[internal_component], + ua_metadata=[ua_component], + api_metadata=[api_component], + os_metadata=[os_component], + language_metadata=[language_component], + env_metadata=[env_component], + config_metadata=[config_component], + feat_metadata=[feat_component], + additional_metadata=[raw_component], + ) + + expected_output = ( + "sdk/python#1.0 md/internal ua/2.0 api/operation#1.1 os/linux#5.4 " + "lang/python#3.12 exec-env/prod cfg/retry-mode#standard ft/paginator " + "raw/string#123" + ) + assert str(user_agent) == expected_output