-
Notifications
You must be signed in to change notification settings - Fork 184
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Codegenerate StructureShape, BlobShape, application, server and Pytho…
…n runtime (#1403) We are introducing code-generation for Python bindings of StructureShape, BlobShape, EnumShape, OperationShape. This PR also add a runtime server implementation to support serving Python business logic directly and with an idiomatic experience. Co-authored-by: david-perez <d@vidp.dev>
- Loading branch information
1 parent
8911e86
commit 8e84ee2
Showing
37 changed files
with
1,436 additions
and
99 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
...n/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCargoDependency.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy | ||
|
|
||
| import software.amazon.smithy.rust.codegen.rustlang.CargoDependency | ||
| import software.amazon.smithy.rust.codegen.rustlang.CratesIo | ||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig | ||
|
|
||
| /** | ||
| * Object used *exclusively* in the runtime of the Python server, for separation concerns. | ||
| * Analogous to the companion object in [CargoDependency] and [ServerCargoDependency]; see its documentation for details. | ||
| * For a dependency that is used in the client, or in both the client and the server, use [CargoDependency] directly. | ||
| */ | ||
| object PythonServerCargoDependency { | ||
| val PyO3: CargoDependency = CargoDependency("pyo3", CratesIo("0.16"), features = setOf("extension-module")) | ||
| val PyO3Asyncio: CargoDependency = CargoDependency("pyo3-asyncio", CratesIo("0.16"), features = setOf("attributes", "tokio-runtime")) | ||
| val Tokio: CargoDependency = CargoDependency("tokio", CratesIo("1.0"), features = setOf("full")) | ||
| val Tracing: CargoDependency = CargoDependency("tracing", CratesIo("0.1")) | ||
| val Tower: CargoDependency = CargoDependency("tower", CratesIo("0.4")) | ||
| val TowerHttp: CargoDependency = CargoDependency("tower-http", CratesIo("0.3"), features = setOf("trace")) | ||
| val Hyper: CargoDependency = CargoDependency("hyper", CratesIo("0.14"), features = setOf("server", "http1", "http2", "tcp", "stream")) | ||
| val NumCpus: CargoDependency = CargoDependency("num_cpus", CratesIo("1.13")) | ||
|
|
||
| fun SmithyHttpServer(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server") | ||
| fun SmithyHttpServerPython(runtimeConfig: RuntimeConfig) = runtimeConfig.runtimeCrate("http-server-python") | ||
| } |
131 changes: 131 additions & 0 deletions
131
...in/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerCodegenVisitor.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
|
|
||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy | ||
|
|
||
| import software.amazon.smithy.build.PluginContext | ||
| import software.amazon.smithy.model.shapes.ServiceShape | ||
| import software.amazon.smithy.model.shapes.StringShape | ||
| import software.amazon.smithy.model.shapes.StructureShape | ||
| import software.amazon.smithy.model.traits.EnumTrait | ||
| import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerEnumGenerator | ||
| import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerServiceGenerator | ||
| import software.amazon.smithy.rust.codegen.server.python.smithy.generators.PythonServerStructureGenerator | ||
| import software.amazon.smithy.rust.codegen.server.smithy.ServerCodegenVisitor | ||
| import software.amazon.smithy.rust.codegen.server.smithy.protocols.ServerProtocolLoader | ||
| import software.amazon.smithy.rust.codegen.smithy.CodegenContext | ||
| import software.amazon.smithy.rust.codegen.smithy.DefaultPublicModules | ||
| import software.amazon.smithy.rust.codegen.smithy.RustCrate | ||
| import software.amazon.smithy.rust.codegen.smithy.SymbolVisitorConfig | ||
| import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.BuilderGenerator | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.CodegenTarget | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.implBlock | ||
| import software.amazon.smithy.rust.codegen.util.getTrait | ||
|
|
||
| /** | ||
| * Entrypoint for Python server-side code generation. This class will walk the in-memory model and | ||
| * generate all the needed types by calling the accept() function on the available shapes. | ||
| * | ||
| * This class inherits from [ServerCodegenVisitor] since it uses most of the functionlities of the super class | ||
| * and have to override the symbol provider with [PythonServerSymbolProvider]. | ||
| */ | ||
| class PythonServerCodegenVisitor(context: PluginContext, codegenDecorator: RustCodegenDecorator) : | ||
| ServerCodegenVisitor(context, codegenDecorator) { | ||
|
|
||
| init { | ||
| val symbolVisitorConfig = | ||
| SymbolVisitorConfig( | ||
| runtimeConfig = settings.runtimeConfig, | ||
| codegenConfig = settings.codegenConfig, | ||
| handleRequired = true | ||
| ) | ||
| val baseModel = baselineTransform(context.model) | ||
| val service = settings.getService(baseModel) | ||
| val (protocol, generator) = | ||
| ServerProtocolLoader( | ||
| codegenDecorator.protocols( | ||
| service.id, | ||
| ServerProtocolLoader.DefaultProtocols | ||
| ) | ||
| ) | ||
| .protocolFor(context.model, service) | ||
| protocolGeneratorFactory = generator | ||
| model = generator.transformModel(codegenDecorator.transformModel(service, baseModel)) | ||
| val baseProvider = PythonCodegenServerPlugin.baseSymbolProvider(model, service, symbolVisitorConfig) | ||
| // Override symbolProvider. | ||
| symbolProvider = | ||
| codegenDecorator.symbolProvider(generator.symbolProvider(model, baseProvider)) | ||
|
|
||
| // Override `codegenContext` which carries the symbolProvider. | ||
| codegenContext = CodegenContext(model, symbolProvider, service, protocol, settings, target = CodegenTarget.SERVER) | ||
|
|
||
| // Override `rustCrate` which carries the symbolProvider. | ||
| rustCrate = RustCrate(context.fileManifest, symbolProvider, DefaultPublicModules, settings.codegenConfig) | ||
| // Override `protocolGenerator` which carries the symbolProvider. | ||
| protocolGenerator = protocolGeneratorFactory.buildProtocolGenerator(codegenContext) | ||
| } | ||
|
|
||
| /** | ||
| * Structure Shape Visitor | ||
| * | ||
| * For each structure shape, generate: | ||
| * - A Rust structure for the shape ([StructureGenerator]). | ||
| * - `pyo3::PyClass` trait implementation. | ||
| * - A builder for the shape. | ||
| * | ||
| * This function _does not_ generate any serializers. | ||
| */ | ||
| override fun structureShape(shape: StructureShape) { | ||
| logger.info("[python-server-codegen] Generating a structure $shape") | ||
| rustCrate.useShapeWriter(shape) { writer -> | ||
| // Use Python specific structure generator that adds the #[pyclass] attribute | ||
| // and #[pymethods] implementation. | ||
| PythonServerStructureGenerator(model, symbolProvider, writer, shape).render(CodegenTarget.SERVER) | ||
| val builderGenerator = | ||
| BuilderGenerator(codegenContext.model, codegenContext.symbolProvider, shape) | ||
| builderGenerator.render(writer) | ||
| writer.implBlock(shape, symbolProvider) { | ||
| builderGenerator.renderConvenienceMethod(this) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * String Shape Visitor | ||
| * | ||
| * Although raw strings require no code generation, enums are actually [EnumTrait] applied to string shapes. | ||
| */ | ||
| override fun stringShape(shape: StringShape) { | ||
| logger.info("[rust-server-codegen] Generating an enum $shape") | ||
| shape.getTrait<EnumTrait>()?.also { enum -> | ||
| rustCrate.useShapeWriter(shape) { writer -> | ||
| PythonServerEnumGenerator(model, symbolProvider, writer, shape, enum, codegenContext.runtimeConfig).render() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Generate service-specific code for the model: | ||
| * - Serializers | ||
| * - Deserializers | ||
| * - Trait implementations | ||
| * - Protocol tests | ||
| * - Operation structures | ||
| * - Python operation handlers | ||
| */ | ||
| override fun serviceShape(shape: ServiceShape) { | ||
| logger.info("[python-server-codegen] Generating a service $shape") | ||
| PythonServerServiceGenerator( | ||
| rustCrate, | ||
| protocolGenerator, | ||
| protocolGeneratorFactory.support(), | ||
| protocolGeneratorFactory.protocol(codegenContext).httpBindingResolver, | ||
| codegenContext, | ||
| ) | ||
| .render() | ||
| } | ||
| } |
26 changes: 26 additions & 0 deletions
26
...otlin/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerRuntimeType.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy | ||
|
|
||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig | ||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeType | ||
|
|
||
| /** | ||
| * Object used *exclusively* in the runtime of the Python server, for separation concerns. | ||
| * Analogous to the companion object in [RuntimeType] and [ServerRuntimeType]; see its documentation for details. | ||
| * For a runtime type that is used in the client, or in both the client and the server, use [RuntimeType] directly. | ||
| */ | ||
| object PythonServerRuntimeType { | ||
|
|
||
| fun SharedSocket(runtimeConfig: RuntimeConfig) = | ||
| RuntimeType("SharedSocket", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python") | ||
|
|
||
| fun Blob(runtimeConfig: RuntimeConfig) = | ||
| RuntimeType("Blob", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python::types") | ||
|
|
||
| fun PyError(runtimeConfig: RuntimeConfig) = | ||
| RuntimeType("Error", PythonServerCargoDependency.SmithyHttpServerPython(runtimeConfig), "${runtimeConfig.crateSrcPrefix}_http_server_python") | ||
| } |
44 changes: 44 additions & 0 deletions
44
...in/software/amazon/smithy/rust/codegen/server/python/smithy/PythonServerSymbolProvider.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy | ||
|
|
||
| import software.amazon.smithy.codegen.core.Symbol | ||
| import software.amazon.smithy.model.shapes.Shape | ||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeType | ||
| import software.amazon.smithy.rust.codegen.smithy.RustSymbolProvider | ||
| import software.amazon.smithy.rust.codegen.smithy.WrappingSymbolProvider | ||
| import software.amazon.smithy.rust.codegen.smithy.rustType | ||
|
|
||
| /** | ||
| * Input / output / error structures can refer to complex types like the ones implemented inside | ||
| * `aws_smithy_types` (a good example is `aws_smithy_types::Blob`). | ||
| * `aws_smithy_http_server_python::types` wraps those types that do not implement directly the | ||
| * `pyo3::PyClass` trait and cannot be shared safely with Python, providing an idiomatic Python / Rust API. | ||
| * | ||
| * This symbol provider ensures types not implementing `pyo3::PyClass` are swapped with their wrappers from | ||
| * `aws_smithy_http_server_python::types`. | ||
| */ | ||
| class PythonServerSymbolProvider(private val base: RustSymbolProvider) : | ||
| WrappingSymbolProvider(base) { | ||
|
|
||
| private val runtimeConfig = config().runtimeConfig | ||
|
|
||
| /** | ||
| * Convert a shape to a Symbol. | ||
| * | ||
| * Swap the shape's symbol if its associated type does not implement `pyo3::PyClass`. | ||
| */ | ||
| override fun toSymbol(shape: Shape): Symbol { | ||
| return when (base.toSymbol(shape).rustType()) { | ||
| RuntimeType.Blob(runtimeConfig).toSymbol().rustType() -> { | ||
| PythonServerRuntimeType.Blob(runtimeConfig).toSymbol() | ||
| } | ||
| else -> { | ||
| base.toSymbol(shape) | ||
| } | ||
| } | ||
| } | ||
| } |
88 changes: 88 additions & 0 deletions
88
...n/smithy/rust/codegen/server/python/smithy/customizations/PythonServerCodegenDecorator.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package software.amazon.smithy.rust.codegen.server.python.smithy.customizations | ||
|
|
||
| import software.amazon.smithy.rust.codegen.rustlang.Writable | ||
| import software.amazon.smithy.rust.codegen.rustlang.docs | ||
| import software.amazon.smithy.rust.codegen.rustlang.rust | ||
| import software.amazon.smithy.rust.codegen.rustlang.rustBlock | ||
| import software.amazon.smithy.rust.codegen.rustlang.writable | ||
| import software.amazon.smithy.rust.codegen.server.python.smithy.PythonServerRuntimeType | ||
| import software.amazon.smithy.rust.codegen.server.smithy.customizations.AddInternalServerErrorToAllOperationsDecorator | ||
| import software.amazon.smithy.rust.codegen.smithy.CodegenContext | ||
| import software.amazon.smithy.rust.codegen.smithy.RuntimeConfig | ||
| import software.amazon.smithy.rust.codegen.smithy.customize.CombinedCodegenDecorator | ||
| import software.amazon.smithy.rust.codegen.smithy.customize.RustCodegenDecorator | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.LibRsCustomization | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.LibRsSection | ||
| import software.amazon.smithy.rust.codegen.smithy.generators.ManifestCustomizations | ||
|
|
||
| /** | ||
| * Configure the [lib] section of `Cargo.toml`. | ||
| * | ||
| * [lib] | ||
| * name = "$CRATE_NAME" | ||
| * crate-type = ["cdylib"] | ||
| */ | ||
| class CdylibManifestDecorator : RustCodegenDecorator { | ||
| override val name: String = "CdylibDecorator" | ||
| override val order: Byte = 0 | ||
|
|
||
| override fun crateManifestCustomizations( | ||
| codegenContext: CodegenContext | ||
| ): ManifestCustomizations = | ||
| mapOf("lib" to mapOf("name" to codegenContext.settings.moduleName, "crate-type" to listOf("cdylib"))) | ||
| } | ||
|
|
||
| /** | ||
| * Add `pub use aws_smithy_http_server_python::types::$TYPE` to lib.rs. | ||
| */ | ||
| class PubUsePythonTypes(private val runtimeConfig: RuntimeConfig) : LibRsCustomization() { | ||
| override fun section(section: LibRsSection): Writable { | ||
| return when (section) { | ||
| is LibRsSection.Body -> writable { | ||
| docs("Re-exported Python types from supporting crates.") | ||
| rustBlock("pub mod python_types") { | ||
| rust("pub use #T;", PythonServerRuntimeType.Blob(runtimeConfig).toSymbol()) | ||
| } | ||
| } | ||
| else -> emptySection | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Decorator applying the customization from [PubUsePythonTypes] class. | ||
| */ | ||
| class PubUsePythonTypesDecorator : RustCodegenDecorator { | ||
| override val name: String = "PubUsePythonTypesDecorator" | ||
| override val order: Byte = 0 | ||
|
|
||
| override fun libRsCustomizations( | ||
| codegenContext: CodegenContext, | ||
| baseCustomizations: List<LibRsCustomization> | ||
| ): List<LibRsCustomization> { | ||
| return baseCustomizations + PubUsePythonTypes(codegenContext.runtimeConfig) | ||
| } | ||
| } | ||
|
|
||
| val DECORATORS = listOf( | ||
| /** | ||
| * Add the [InternalServerError] error to all operations. | ||
| * This is done because the Python interpreter can raise exceptions during execution | ||
| */ | ||
| AddInternalServerErrorToAllOperationsDecorator(), | ||
| // Add the [lib] section to Cargo.toml to configure the generation of the shared library: | ||
| CdylibManifestDecorator(), | ||
| // Add `pub use` of `aws_smithy_http_server_python::types`. | ||
| PubUsePythonTypesDecorator() | ||
| ) | ||
|
|
||
| // Combined codegen decorator for Python services. | ||
| class PythonServerCodegenDecorator : CombinedCodegenDecorator(DECORATORS) { | ||
| override val name: String = "PythonServerCodegenDecorator" | ||
| override val order: Byte = -1 | ||
| } |
Oops, something went wrong.