diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d594bf6e..7d4b14da0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,11 @@ jobs: activate-environment: true enable-cache: true + - name: Install pandoc + uses: pandoc/actions/setup@v1 + with: + version: 3.8.2 + - name: Setup workspace run: | make install diff --git a/README.md b/README.md index 502d20217..de459820e 100644 --- a/README.md +++ b/README.md @@ -112,12 +112,13 @@ With both files your project directory should look like this: The code generator libraries have not been published yet, so you'll need to build it yourself. To build and run the generator, you will need -the following prerequisites: +the following prerequisites installed in your environment: * [uv](https://docs.astral.sh/uv/) * The [Smithy CLI](https://smithy.io/2.0/guides/smithy-cli/cli_installation.html) * JDK 17 or newer * make +* [pandoc](https://pandoc.org/installing.html) CLI This project uses [uv](https://docs.astral.sh/uv/) for managing all things python. Once you have it installed, run the following command to check that it's ready to use: @@ -169,6 +170,12 @@ if __name__ == "__main__": asyncio.run(main()) ``` +#### pandoc CLI + +The code generator uses [pandoc](https://pandoc.org/) to convert documentation from Smithy models +into Markdown format for Python docstrings, which can then be used to generate +documentation with [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/getting-started/). + #### Is Java really required? Only for now. Once the generator has been published, the Smithy CLI will be able diff --git a/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java index 5bfbdba22..d7546fe1e 100644 --- a/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java +++ b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java @@ -4,6 +4,9 @@ */ package software.amazon.smithy.python.codegen.test; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; import java.nio.file.Path; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -14,7 +17,8 @@ import software.amazon.smithy.python.codegen.PythonClientCodegenPlugin; /** - * Simple test that executes the Python client codegen plugin for an AWS-like service. + * Simple test that executes the Python client codegen plugin for an AWS-like service + * and validates that MkDocs documentation files are generated. */ public class AwsCodegenTest { @@ -36,6 +40,20 @@ public void testCodegen(@TempDir Path tempDir) { .model(model) .build(); plugin.execute(context); + + // Verify MkDocs documentation files are generated for AWS services + Path docsDir = tempDir.resolve("docs"); + assertTrue(Files.exists(docsDir), "docs directory should be created"); + + Path indexFile = docsDir.resolve("index.md"); + assertTrue(Files.exists(indexFile), "index.md should be generated for AWS services"); + + Path operationsDir = docsDir.resolve("operations"); + assertTrue(Files.exists(operationsDir), "operations directory should be created"); + + // Verify at least one operation file exists + Path basicOperationFile = operationsDir.resolve("basic_operation.md"); + assertTrue(Files.exists(basicOperationFile), "basic_operation.md should be generated"); } } diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsMkDocsFileGenerator.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsMkDocsFileGenerator.java new file mode 100644 index 000000000..7ca77ba14 --- /dev/null +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsMkDocsFileGenerator.java @@ -0,0 +1,525 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.aws.codegen; + +import static software.amazon.smithy.python.codegen.SymbolProperties.OPERATION_METHOD; + +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; +import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.model.knowledge.EventStreamIndex; +import software.amazon.smithy.model.traits.InputTrait; +import software.amazon.smithy.model.traits.OutputTrait; +import software.amazon.smithy.model.traits.StringTrait; +import software.amazon.smithy.model.traits.TitleTrait; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.integrations.PythonIntegration; +import software.amazon.smithy.python.codegen.sections.*; +import software.amazon.smithy.python.codegen.writer.PythonWriter; +import software.amazon.smithy.utils.CodeInterceptor; +import software.amazon.smithy.utils.CodeSection; + +/** + * Generates API Reference stub files in Markdown for MkDocs/mkdocstrings doc gen. + * + * This integration creates individual .md files for operations, structures, errors, + * unions, and enums that are used with MkDocs and mkdocstrings to generate documentation. + * It also generates a comprehensive index.md that documents the client and config objects, + * along with lists of all available client operations and models. + */ +public class AwsMkDocsFileGenerator implements PythonIntegration { + + // Shared collections to track what we've generated + private final TreeSet operations = new TreeSet<>(); + private final TreeSet structures = new TreeSet<>(); + private final TreeSet errors = new TreeSet<>(); + private final TreeSet enums = new TreeSet<>(); + private final TreeSet unions = new TreeSet<>(); + + @Override + public List> interceptors( + GenerationContext context + ) { + if (!CodegenUtils.isAwsService(context)) { + return List.of(); + } + var interceptors = new ArrayList>(); + interceptors.add(new OperationGenerationInterceptor(context, operations)); + interceptors.add(new StructureGenerationInterceptor(context, structures)); + interceptors.add(new ErrorGenerationInterceptor(context, errors)); + interceptors.add(new EnumGenerationInterceptor(context, enums)); + interceptors.add(new IntEnumGenerationInterceptor(context, enums)); + interceptors.add(new UnionGenerationInterceptor(context, unions)); + return interceptors; + } + + @Override + public void customize(GenerationContext context) { + if (!CodegenUtils.isAwsService(context)) { + return; + } + // This runs after shape generation, so we can now generate the index with all collected operations/models + new IndexGenerator(context, operations, structures, errors, enums, unions).run(); + } + + /** + * Utility method to generate a header for documentation files. + * + * @param title The title of the section. + * @return A formatted header string. + */ + private static String generateHeader(String title) { + return String.format("# %s%n%n", title); + } + + private static final class OperationGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + private final TreeSet operations; + + public OperationGenerationInterceptor( + GenerationContext context, + TreeSet operations + ) { + this.context = context; + this.operations = operations; + } + + @Override + public Class sectionType() { + return OperationSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, OperationSection section) { + var operation = section.operation(); + var operationSymbol = context.symbolProvider().toSymbol(operation).expectProperty(OPERATION_METHOD); + var input = context.model().expectShape(operation.getInputShape()); + var inputSymbol = context.symbolProvider().toSymbol(input); + var output = context.model().expectShape(operation.getOutputShape()); + var outputSymbol = context.symbolProvider().toSymbol(output); + + String operationName = operationSymbol.getName(); + String clientName = context.symbolProvider().toSymbol(section.service()).getName(); + String docsFileName = String.format("docs/operations/%s.md", operationName); + String fullOperationReference = String.format("%s.client.%s.%s", + context.settings().moduleName(), + clientName, + operationName); + + // Track this operation + operations.add(operationName); + + context.writerDelegator().useFileWriter(docsFileName, "", fileWriter -> { + fileWriter.write(generateHeader(operationName)); + + fileWriter.write("\n## Operation\n\n"); + fileWriter.write("::: " + fullOperationReference); + fileWriter.write(""" + options: + heading_level: 3 + """); + + // Add Input Structure documentation + fileWriter.write("\n## Input\n\n"); + fileWriter.write("::: " + inputSymbol.toString()); + fileWriter.write(""" + options: + heading_level: 3 + """); + + // Add Output Structure documentation + fileWriter.write("\n## Output\n\n"); + + // Check if operation returns event streams + var eventStreamIndex = EventStreamIndex.of(context.model()); + var inputStreamInfo = eventStreamIndex.getInputInfo(operation); + var outputStreamInfo = eventStreamIndex.getOutputInfo(operation); + boolean hasInputStream = inputStreamInfo.isPresent(); + boolean hasOutputStream = outputStreamInfo.isPresent(); + + if (hasInputStream && hasOutputStream) { + // Duplex event stream + var inputStreamTarget = inputStreamInfo.get().getEventStreamTarget(); + var inputStreamShape = context.model().expectShape(inputStreamTarget.getId()); + var inputStreamSymbol = context.symbolProvider().toSymbol(inputStreamShape); + + var outputStreamTarget = outputStreamInfo.get().getEventStreamTarget(); + var outputStreamShape = context.model().expectShape(outputStreamTarget.getId()); + var outputStreamSymbol = context.symbolProvider().toSymbol(outputStreamShape); + + fileWriter.write("This operation returns a `DuplexEventStream` for bidirectional streaming.\n\n"); + fileWriter.write("### Event Stream Structure\n\n"); + fileWriter.write("#### Input Event Type\n\n"); + fileWriter.write("[`$L`](../unions/$L.md)\n\n", + inputStreamSymbol.getName(), + inputStreamSymbol.getName()); + fileWriter.write("#### Output Event Type\n\n"); + fileWriter.write("[`$L`](../unions/$L.md)\n\n", + outputStreamSymbol.getName(), + outputStreamSymbol.getName()); + fileWriter.write("### Initial Response Structure\n\n"); + } else if (hasInputStream) { + // Input event stream + var streamTarget = inputStreamInfo.get().getEventStreamTarget(); + var streamShape = context.model().expectShape(streamTarget.getId()); + var streamSymbol = context.symbolProvider().toSymbol(streamShape); + + fileWriter + .write("This operation returns an `InputEventStream` for client-to-server streaming.\n\n"); + fileWriter.write("### Event Stream Structure\n\n"); + fileWriter.write("#### Input Event Type\n\n"); + fileWriter.write("[`$L`](../unions/$L.md)\n\n", + streamSymbol.getName(), + streamSymbol.getName()); + fileWriter.write("### Final Response Structure\n\n"); + } else if (hasOutputStream) { + var streamTarget = outputStreamInfo.get().getEventStreamTarget(); + var streamShape = context.model().expectShape(streamTarget.getId()); + var streamSymbol = context.symbolProvider().toSymbol(streamShape); + + fileWriter + .write("This operation returns an `OutputEventStream` for server-to-client streaming.\n\n"); + fileWriter.write("### Event Stream Structure\n\n"); + fileWriter.write("#### Output Event Type\n\n"); + fileWriter.write("[`$L`](../unions/$L.md)\n\n", + streamSymbol.getName(), + streamSymbol.getName()); + fileWriter.write("### Initial Response Structure\n\n"); + } + + fileWriter.write("::: " + outputSymbol.toString()); + // Use heading level 4 for event streams, level 3 for regular operations + int headingLevel = (hasInputStream || hasOutputStream) ? 4 : 3; + fileWriter.write(""" + options: + heading_level: $L + """, headingLevel); + + // Add Errors documentation + var operationErrors = operation.getErrorsSet(); + if (!operationErrors.isEmpty()) { + fileWriter.write("\n## Errors\n\n"); + for (var errorId : operationErrors) { + var errorShape = context.model().expectShape(errorId); + var errorSymbol = context.symbolProvider().toSymbol(errorShape); + String errorName = errorSymbol.getName(); + fileWriter.write("- [`$L`](../errors/$L.md)\n", errorName, errorName); + } + } + }); + } + } + + private static final class StructureGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + private final TreeSet structures; + + public StructureGenerationInterceptor(GenerationContext context, TreeSet structures) { + this.context = context; + this.structures = structures; + } + + @Override + public Class sectionType() { + return StructureSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, StructureSection section) { + var shape = section.structure(); + var symbol = context.symbolProvider().toSymbol(shape); + String symbolName = symbol.getName(); + String docsFileName = String.format("docs/structures/%s.md", symbolName); + // Don't generate separate docs for input/output structures (they're included with operations) + if (!shape.hasTrait(InputTrait.class) && !shape.hasTrait(OutputTrait.class)) { + structures.add(symbolName); + context.writerDelegator().useFileWriter(docsFileName, "", writer -> { + writer.write("::: " + symbol.toString()); + writer.write(""" + options: + heading_level: 1 + """); + }); + } + } + } + + private static final class ErrorGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + private final TreeSet errors; + + public ErrorGenerationInterceptor(GenerationContext context, TreeSet errors) { + this.context = context; + this.errors = errors; + } + + @Override + public Class sectionType() { + return ErrorSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, ErrorSection section) { + var symbol = section.errorSymbol(); + String symbolName = symbol.getName(); + String docsFileName = String.format("docs/errors/%s.md", symbolName); + errors.add(symbolName); + context.writerDelegator().useFileWriter(docsFileName, "", writer -> { + writer.write("::: " + symbol.toString()); + writer.write(""" + options: + heading_level: 1 + """); + }); + } + } + + private static final class EnumGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + private final TreeSet enums; + + public EnumGenerationInterceptor(GenerationContext context, TreeSet enums) { + this.context = context; + this.enums = enums; + } + + @Override + public Class sectionType() { + return EnumSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, EnumSection section) { + var symbol = section.enumSymbol(); + String symbolName = symbol.getName(); + String docsFileName = String.format("docs/enums/%s.md", symbolName); + enums.add(symbolName); + context.writerDelegator().useFileWriter(docsFileName, "", writer -> { + writer.write("::: " + symbol.toString()); + writer.write(""" + options: + heading_level: 1 + members: true + """); + }); + } + } + + private static final class IntEnumGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + private final TreeSet enums; + + public IntEnumGenerationInterceptor(GenerationContext context, TreeSet enums) { + this.context = context; + this.enums = enums; + } + + @Override + public Class sectionType() { + return IntEnumSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, IntEnumSection section) { + var symbol = section.enumSymbol(); + String symbolName = symbol.getName(); + String docsFileName = String.format("docs/enums/%s.md", symbolName); + enums.add(symbolName); + context.writerDelegator().useFileWriter(docsFileName, "", writer -> { + writer.write("::: " + symbol.toString()); + writer.write(""" + options: + heading_level: 1 + members: true + """); + }); + } + } + + private static final class UnionGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + private final TreeSet unions; + + public UnionGenerationInterceptor( + GenerationContext context, + TreeSet unions + ) { + this.context = context; + this.unions = unions; + } + + @Override + public Class sectionType() { + return UnionSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, UnionSection section) { + String parentName = section.parentName(); + + String docsFileName = String.format("docs/unions/%s.md", parentName); + unions.add(parentName); + var unionSymbol = context.symbolProvider().toSymbol(section.unionShape()); + context.writerDelegator().useFileWriter(docsFileName, "", writer -> { + // Document the union type alias + writer.write("::: " + unionSymbol.toString()); + writer.write(""" + options: + heading_level: 1 + """); + + // Document each union member on the same page + writer.write("\n## Union Members\n\n"); + for (var member : section.unionShape().members()) { + var memberSymbol = context.symbolProvider().toSymbol(member); + writer.write("::: " + memberSymbol.toString()); + writer.write(""" + options: + heading_level: 3 + """); + } + + // Document the unknown variant + var unknownSymbol = unionSymbol + .expectProperty(software.amazon.smithy.python.codegen.SymbolProperties.UNION_UNKNOWN); + writer.write("::: " + unknownSymbol.toString()); + writer.write(""" + options: + heading_level: 3 + """); + }); + } + } + + /** + * Generates the main index.md file and config documentation after all shapes are generated. + */ + private static final class IndexGenerator implements Runnable { + + private final GenerationContext context; + private final TreeSet operations; + private final TreeSet structures; + private final TreeSet errors; + private final TreeSet enums; + private final TreeSet unions; + + public IndexGenerator( + GenerationContext context, + TreeSet operations, + TreeSet structures, + TreeSet errors, + TreeSet enums, + TreeSet unions + ) { + this.context = context; + this.operations = operations; + this.structures = structures; + this.errors = errors; + this.enums = enums; + this.unions = unions; + } + + @Override + public void run() { + var service = context.model().expectShape(context.settings().service()); + String clientName = context.symbolProvider().toSymbol(service).getName(); + String title = service.getTrait(ServiceTrait.class) + .map(ServiceTrait::getSdkId) + .orElseGet(() -> service.getTrait(TitleTrait.class) + .map(StringTrait::getValue) + .orElse(clientName)); + String moduleName = context.settings().moduleName(); + + // Generate main index.md + context.writerDelegator().useFileWriter("docs/index.md", "", indexWriter -> { + indexWriter.write("# $L\n\n", title); + + // Client section + indexWriter.write("\n## Client\n\n"); + indexWriter.write("::: $L.client.$L", moduleName, clientName); + indexWriter.write(""" + options: + merge_init_into_class: true + docstring_options: + ignore_init_summary: true + members: false + heading_level: 3 + """); + + // Operations section + indexWriter.write("\n## Available Operations\n\n"); + for (String operation : operations) { + indexWriter.write("- [`$L`](operations/$L.md)\n", operation, operation); + } + + // Config section + indexWriter.write("\n## Configuration\n\n"); + indexWriter.write("::: $L.config.Config", moduleName); + indexWriter.write(""" + options: + merge_init_into_class: true + docstring_options: + ignore_init_summary: true + heading_level: 3 + """); + + // Errors section + if (!errors.isEmpty()) { + indexWriter.write("\n## Errors\n\n"); + for (String error : errors) { + indexWriter.write("- [`$L`](errors/$L.md)\n", error, error); + } + } + + // Structures section + if (!structures.isEmpty()) { + indexWriter.write("\n## Structures\n\n"); + for (String structure : structures) { + indexWriter.write("- [`$L`](structures/$L.md)\n", structure, structure); + } + } + + // Unions section + if (!unions.isEmpty()) { + indexWriter.write("\n## Unions\n\n"); + for (String union : unions) { + indexWriter.write("- [`$L`](unions/$L.md)\n", union, union); + } + } + + // Enums section + if (!enums.isEmpty()) { + indexWriter.write("\n## Enums\n\n"); + for (String enumName : enums) { + indexWriter.write("- [`$L`](enums/$L.md)\n", enumName, enumName); + } + } + }); + + // Generate custom CSS file to widen content area width + context.writerDelegator().useFileWriter("docs/stylesheets/extra.css", "", cssWriter -> { + cssWriter.write(""" + .md-grid { + max-width: 70rem; + } + """); + }); + } + } +} diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java deleted file mode 100644 index 0141801b3..000000000 --- a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.aws.codegen; - -import static software.amazon.smithy.python.codegen.SymbolProperties.OPERATION_METHOD; - -import java.util.List; -import software.amazon.smithy.model.traits.InputTrait; -import software.amazon.smithy.model.traits.OutputTrait; -import software.amazon.smithy.python.codegen.GenerationContext; -import software.amazon.smithy.python.codegen.integrations.PythonIntegration; -import software.amazon.smithy.python.codegen.sections.*; -import software.amazon.smithy.python.codegen.writer.PythonWriter; -import software.amazon.smithy.utils.CodeInterceptor; -import software.amazon.smithy.utils.CodeSection; - -public class AwsRstDocFileGenerator implements PythonIntegration { - - @Override - public List> interceptors( - GenerationContext context - ) { - return List.of( - // We generate custom RST files for each member that we want to have - // its own page. This gives us much more fine-grained control of - // what gets generated than just using automodule or autoclass on - // the client would alone. - new OperationGenerationInterceptor(context), - new StructureGenerationInterceptor(context), - new ErrorGenerationInterceptor(context), - new UnionGenerationInterceptor(context), - new UnionMemberGenerationInterceptor(context)); - } - - /** - * Utility method to generate a header for documentation files. - * - * @param title The title of the section. - * @return A formatted header string. - */ - private static String generateHeader(String title) { - return String.format("%s%n%s%n%n", title, "=".repeat(title.length())); - } - - private static final class OperationGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public OperationGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return OperationSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, OperationSection section) { - var operation = section.operation(); - var operationSymbol = context.symbolProvider().toSymbol(operation).expectProperty(OPERATION_METHOD); - var input = context.model().expectShape(operation.getInputShape()); - var inputSymbol = context.symbolProvider().toSymbol(input); - var output = context.model().expectShape(operation.getOutputShape()); - var outputSymbol = context.symbolProvider().toSymbol(output); - - String operationName = operationSymbol.getName(); - String inputSymbolName = inputSymbol.toString(); - String outputSymbolName = outputSymbol.toString(); - String serviceName = context.symbolProvider().toSymbol(section.service()).getName(); - String docsFileName = String.format("docs/client/%s.rst", operationName); - String fullOperationReference = String.format("%s.client.%s.%s", - context.settings().moduleName(), - serviceName, - operationName); - - context.writerDelegator().useFileWriter(docsFileName, "", fileWriter -> { - fileWriter.write(generateHeader(operationName)); - fileWriter.write(".. automethod:: " + fullOperationReference + "\n\n"); - fileWriter.write(".. toctree::\n :hidden:\n :maxdepth: 2\n\n"); - fileWriter.write("=================\nInput:\n=================\n\n"); - fileWriter.write(".. autoclass:: " + inputSymbolName + "\n :members:\n"); - fileWriter.write("=================\nOutput:\n=================\n\n"); - fileWriter.write(".. autoclass:: " + outputSymbolName + "\n :members:\n"); - }); - } - } - - private static final class StructureGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public StructureGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return StructureSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, StructureSection section) { - var shape = section.structure(); - var symbol = context.symbolProvider().toSymbol(shape); - String docsFileName = String.format("docs/models/%s.rst", - symbol.getName()); - if (!shape.hasTrait(InputTrait.class) && !shape.hasTrait(OutputTrait.class)) { - context.writerDelegator().useFileWriter(docsFileName, "", writer -> { - writer.write(generateHeader(symbol.getName())); - writer.write(".. autoclass:: " + symbol.toString() + "\n :members:\n"); - }); - } - } - } - - private static final class ErrorGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public ErrorGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return ErrorSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, ErrorSection section) { - var symbol = section.errorSymbol(); - String docsFileName = String.format("docs/models/%s.rst", - symbol.getName()); - context.writerDelegator().useFileWriter(docsFileName, "", writer -> { - writer.write(generateHeader(symbol.getName())); - writer.write(".. autoexception:: " + symbol.toString() + "\n :members:\n :show-inheritance:\n"); - }); - } - } - - private static final class UnionGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public UnionGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return UnionSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, UnionSection section) { - String parentName = section.parentName(); - String docsFileName = String.format("docs/models/%s.rst", parentName); - context.writerDelegator().useFileWriter(docsFileName, "", writer -> { - writer.write(".. _" + parentName + ":\n\n"); - writer.write(generateHeader(parentName)); - writer.write( - ".. autodata:: " + context.symbolProvider().toSymbol(section.unionShape()).toString() + " \n"); - }); - } - } - - private static final class UnionMemberGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public UnionMemberGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return UnionMemberSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, UnionMemberSection section) { - var memberSymbol = section.memberSymbol(); - String symbolName = memberSymbol.getName(); - String docsFileName = String.format("docs/models/%s.rst", symbolName); - context.writerDelegator().useFileWriter(docsFileName, "", writer -> { - writer.write(".. _" + symbolName + ":\n\n"); - writer.write(generateHeader(symbolName)); - writer.write(".. autoclass:: " + memberSymbol.toString() + " \n"); - }); - } - } -} diff --git a/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index 2a8cfe6d6..a0b3d479b 100644 --- a/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -8,4 +8,4 @@ software.amazon.smithy.python.aws.codegen.AwsProtocolsIntegration software.amazon.smithy.python.aws.codegen.AwsServiceIdIntegration software.amazon.smithy.python.aws.codegen.AwsUserAgentIntegration software.amazon.smithy.python.aws.codegen.AwsStandardRegionalEndpointsIntegration -software.amazon.smithy.python.aws.codegen.AwsRstDocFileGenerator +software.amazon.smithy.python.aws.codegen.AwsMkDocsFileGenerator diff --git a/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java b/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java deleted file mode 100644 index 8f4a7bd73..000000000 --- a/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.aws.codegen; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import software.amazon.smithy.python.codegen.writer.MarkdownToRstDocConverter; - -public class MarkdownToRstDocConverterTest { - - private MarkdownToRstDocConverter markdownToRstDocConverter; - - @BeforeEach - public void setUp() { - markdownToRstDocConverter = MarkdownToRstDocConverter.getInstance(); - } - - @Test - public void testConvertCommonmarkToRstWithTitleAndParagraph() { - String html = "

Title

Paragraph

"; - String expected = "Title\n=====\nParagraph"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithImportantNote() { - String html = "Important note"; - String expected = ".. important::\n Important note"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithList() { - String html = "
  • Item 1
  • Item 2
"; - String expected = "* Item 1\n* Item 2"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithMixedElements() { - String html = "

Title

Paragraph

  • Item 1
  • Item 2
"; - String expected = "Title\n=====\nParagraph\n\n\n* Item 1\n* Item 2"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithNestedElements() { - String html = "

Title

Paragraph with bold text

"; - String expected = "Title\n=====\nParagraph with **bold** text"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithAnchorTag() { - String html = "Link"; - String expected = "`Link `_"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithBoldTag() { - String html = "Bold text"; - String expected = "**Bold text**"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithItalicTag() { - String html = "Italic text"; - String expected = "*Italic text*"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithCodeTag() { - String html = "code snippet"; - String expected = "``code snippet``"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithNoteTag() { - String html = "Note text"; - String expected = ".. note::\n Note text"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithNestedList() { - String html = "
  • Item 1
    • Subitem 1
  • Item 2
"; - String expected = "* Item 1\n\n * Subitem 1\n* Item 2"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithFormatSpecifierCharacters() { - // Test that Smithy format specifier characters ($) are properly escaped and treated as literal text - String html = "

Testing $placeholder_one and $placeholder_two

"; - String expected = "Testing $placeholder_one and $placeholder_two"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } -} diff --git a/codegen/buildSrc/src/main/kotlin/smithy-python.java-conventions.gradle.kts b/codegen/buildSrc/src/main/kotlin/smithy-python.java-conventions.gradle.kts index f50917c67..9dd1a1f95 100644 --- a/codegen/buildSrc/src/main/kotlin/smithy-python.java-conventions.gradle.kts +++ b/codegen/buildSrc/src/main/kotlin/smithy-python.java-conventions.gradle.kts @@ -39,6 +39,7 @@ dependencies { testRuntimeOnly(libs.junit.jupiter.engine) testRuntimeOnly(libs.junit.platform.launcher) testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mockito.core) compileOnly("com.github.spotbugs:spotbugs-annotations:${spotbugs.toolVersion.get()}") testCompileOnly("com.github.spotbugs:spotbugs-annotations:${spotbugs.toolVersion.get()}") } diff --git a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java index 929aeb602..f9b98789c 100644 --- a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java +++ b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java @@ -4,6 +4,9 @@ */ package software.amazon.smithy.python.codegen.test; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.nio.file.Files; import java.nio.file.Path; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -14,8 +17,8 @@ import software.amazon.smithy.python.codegen.PythonClientCodegenPlugin; /** - * Simple test that executes the Python client codegen plugin. Currently, this is about as much "testing" as - * we can do, aside from the protocol tests. JUnit will set up and tear down a tempdir to house the codegen artifacts. + * Simple test that executes the Python client codegen plugin for a non-AWS service + * and validates that MkDocs documentation files are NOT generated. */ public class PythonCodegenTest { @@ -38,5 +41,9 @@ public void testCodegen(@TempDir Path tempDir) { .model(model) .build(); plugin.execute(context); + + // Verify MkDocs files are NOT generated for non-AWS services + Path docsDir = tempDir.resolve("docs"); + assertFalse(Files.exists(docsDir), "docs directory should NOT be created for non-AWS services"); } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 026e4dfe2..784173e14 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -23,9 +23,9 @@ import software.amazon.smithy.python.codegen.integrations.PythonIntegration; import software.amazon.smithy.python.codegen.integrations.RuntimeClientPlugin; import software.amazon.smithy.python.codegen.sections.*; -import software.amazon.smithy.python.codegen.writer.MarkdownToRstDocConverter; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; +import software.amazon.smithy.utils.StringUtils; /** * Generates the actual client and implements operations. @@ -60,18 +60,7 @@ private void generateService(PythonWriter writer) { var docs = service.getTrait(DocumentationTrait.class) .map(StringTrait::getValue) .orElse("Client for " + serviceSymbol.getName()); - String rstDocs = - MarkdownToRstDocConverter.getInstance().convertCommonmarkToRst(docs); - writer.writeDocs(() -> { - writer.write(""" - $L - - :param config: Optional configuration for the client. Here you can set things like the - endpoint for HTTP services or auth credentials. - - :param plugins: A list of callables that modify the configuration dynamically. These - can be used to set defaults, for example.""", rstDocs); - }); + writer.writeDocs(docs, context); var defaultPlugins = new LinkedHashSet(); @@ -85,17 +74,22 @@ private void generateService(PythonWriter writer) { writer.write(""" def __init__(self, config: $1T | None = None, plugins: list[$2T] | None = None): + $3C self._config = config or $1T() client_plugins: list[$2T] = [ - $3C + $4C ] if plugins: client_plugins.extend(plugins) for plugin in client_plugins: plugin(self._config) - """, configSymbol, pluginSymbol, writer.consumer(w -> writeDefaultPlugins(w, defaultPlugins))); + """, + configSymbol, + pluginSymbol, + writer.consumer(w -> writeConstructorDocs(w, serviceSymbol.getName())), + writer.consumer(w -> writeDefaultPlugins(w, defaultPlugins))); var topDownIndex = TopDownIndex.of(model); var eventStreamIndex = EventStreamIndex.of(model); @@ -116,6 +110,25 @@ private void writeDefaultPlugins(PythonWriter writer, Collection { + writer.writeInline(""" + Constructor for `$L`. + + Args: + config: + Optional configuration for the client. Here you can set things like + the endpoint for HTTP services or auth credentials. + plugins: + A list of callables that modify the configuration dynamically. These + can be used to set defaults, for example. + """, clientName); + }); + } + /** * Generates the function for a single operation. */ @@ -144,33 +157,50 @@ private void generateOperation(PythonWriter writer, OperationShape operation) { ${C|} return await pipeline(call) """, - writer.consumer(w -> writeSharedOperationInit(w, operation, input))); + writer.consumer(w -> writeSharedOperationInit(w, operation, input, output))); writer.popState(); } - private void writeSharedOperationInit(PythonWriter writer, OperationShape operation, Shape input) { + private void writeSharedOperationInit(PythonWriter writer, OperationShape operation, Shape input, Shape output) { + writeSharedOperationInit(writer, operation, input, output, null); + } + + private void writeSharedOperationInit( + PythonWriter writer, + OperationShape operation, + Shape input, + Shape output, + String customReturnDocs + ) { writer.writeDocs(() -> { - var docs = writer.formatDocs(operation.getTrait(DocumentationTrait.class) + var inputSymbolName = symbolProvider.toSymbol(input).getName(); + var outputSymbolName = symbolProvider.toSymbol(output).getName(); + + var operationDocs = writer.formatDocs(operation.getTrait(DocumentationTrait.class) .map(StringTrait::getValue) .orElse(String.format("Invokes the %s operation.", - operation.getId().getName()))); + operation.getId().getName())), + context); - var inputDocs = input.getTrait(DocumentationTrait.class) - .map(StringTrait::getValue) - .orElse("The operation's input."); + var inputDocs = String.format("An instance of `%s`.", inputSymbolName); + var outputDocs = customReturnDocs != null ? customReturnDocs + : String.format("An instance of `%s`.", outputSymbolName); - writer.write(""" + writer.writeInline(""" $L - """, docs); - writer.write(""); - writer.write(":param input: $L", inputDocs); - writer.write(""); - writer.write(""" - :param plugins: A list of callables that modify the configuration dynamically. - Changes made by these plugins only apply for the duration of the operation - execution and will not affect any other operation invocations. - """); + Args: + input: + $L + plugins: + A list of callables that modify the configuration dynamically. + Changes made by these plugins only apply for the duration of the + operation execution and will not affect any other operation + invocations. + + Returns: + ${L|} + """, operationDocs, inputDocs, outputDocs); }); var defaultPlugins = new LinkedHashSet(); @@ -260,6 +290,11 @@ private void generateEventStreamOperation(PythonWriter writer, OperationShape op if (inputStreamSymbol.isPresent()) { if (outputStreamSymbol.isPresent()) { writer.addImport("smithy_core.aio.eventstream", "DuplexEventStream"); + var returnDocs = generateEventStreamReturnDocs( + "DuplexEventStream", + inputStreamSymbol.get().getName(), + outputStreamSymbol.get().getName(), + outputSymbol.getName()); writer.write(""" async def ${operationName:L}( self, @@ -274,9 +309,14 @@ private void generateEventStreamOperation(PythonWriter writer, OperationShape op ${outputStreamDeserializer:T}().deserialize ) """, - writer.consumer(w -> writeSharedOperationInit(w, operation, input))); + writer.consumer(w -> writeSharedOperationInit(w, operation, input, output, returnDocs))); } else { writer.addImport("smithy_core.aio.eventstream", "InputEventStream"); + var returnDocs = generateEventStreamReturnDocs( + "InputEventStream", + inputStreamSymbol.get().getName(), + null, + outputSymbol.getName()); writer.write(""" async def ${operationName:L}( self, @@ -288,10 +328,15 @@ private void generateEventStreamOperation(PythonWriter writer, OperationShape op call, ${inputStream:T} ) - """, writer.consumer(w -> writeSharedOperationInit(w, operation, input))); + """, writer.consumer(w -> writeSharedOperationInit(w, operation, input, output, returnDocs))); } } else { writer.addImport("smithy_core.aio.eventstream", "OutputEventStream"); + var returnDocs = generateEventStreamReturnDocs( + "OutputEventStream", + null, + outputStreamSymbol.get().getName(), + outputSymbol.getName()); writer.write(""" async def ${operationName:L}( self, @@ -305,9 +350,42 @@ private void generateEventStreamOperation(PythonWriter writer, OperationShape op ${outputStreamDeserializer:T}().deserialize ) """, - writer.consumer(w -> writeSharedOperationInit(w, operation, input))); + writer.consumer(w -> writeSharedOperationInit(w, operation, input, output, returnDocs))); } writer.popState(); } + + /** + * Generates documentation for event stream return types. + */ + private String generateEventStreamReturnDocs( + String containerType, + String inputStreamName, + String outputStreamName, + String outputName + ) { + String docs = switch (containerType) { + case "DuplexEventStream" -> String.format( + "A `DuplexEventStream` for bidirectional streaming of `%s` and `%s` events with initial `%s` response.", + inputStreamName, + outputStreamName, + outputName); + case "InputEventStream" -> String.format( + "An `InputEventStream` for client-to-server streaming of `%s` events with final `%s` response.", + inputStreamName, + outputName); + case "OutputEventStream" -> String.format( + "An `OutputEventStream` for server-to-client streaming of `%s` events with initial `%s` response.", + outputStreamName, + outputName); + default -> throw new IllegalArgumentException("Unknown event stream type: " + containerType); + }; + // Subtract 12 chars for 3 indentation (4 spaces each) + String wrapped = StringUtils.wrap( + docs, + CodegenUtils.MAX_PREFERRED_LINE_LENGTH - 12); + // Add additional indentation (4 spaces) to continuation lines for proper Google-style formatting + return wrapped.replace("\n", "\n "); + } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java index 994fc9fad..777520c3b 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java @@ -280,6 +280,20 @@ private static ZonedDateTime parseHttpDate(Node value) { return instant.atZone(ZoneId.of("UTC")); } + /** + * Determines whether the service being generated is an AWS service. + * + * AWS services are identified by the presence of the AWS ServiceTrait which + * contains AWS-specific metadata like SDK ID and endpoint prefix. + * + * @param context The generation context. + * @return Returns true if the service is an AWS service, false otherwise. + */ + public static boolean isAwsService(GenerationContext context) { + var service = context.model().expectShape(context.settings().service()); + return service.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class); + } + /** * Writes an accessor for a structure member, handling defaultedness and nullability. * diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java index 83f1258f4..c8fa203c4 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java @@ -587,7 +587,7 @@ private void writeTestBlock( writer.write("@mark.xfail()"); } writer.openBlock("async def test_$L() -> None:", "", CaseUtils.toSnakeCase(testName), () -> { - testCase.getDocumentation().ifPresent(writer::writeDocs); + testCase.getDocumentation().ifPresent(docs -> writer.writeDocs(docs, context)); f.run(); }); } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java index 654bea9fe..3f97a60c1 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java @@ -101,20 +101,29 @@ public final class SmithyPythonDependency { false); /** - * library used for documentation generation + * MkDocs documentation generator */ - public static final PythonDependency SPHINX = new PythonDependency( - "sphinx", - ">=8.2.3", + public static final PythonDependency MKDOCS = new PythonDependency( + "mkdocs", + "~=1.6.1", Type.DOCS_DEPENDENCY, false); /** - * sphinx theme + * mkdocstrings plugin for auto-generating documentation from docstrings */ - public static final PythonDependency SPHINX_PYDATA_THEME = new PythonDependency( - "pydata-sphinx-theme", - ">=0.16.1", + public static final PythonDependency MKDOCSTRINGS = new PythonDependency( + "mkdocstrings[python]", + "~=0.30.1", + Type.DOCS_DEPENDENCY, + false); + + /** + * Material theme for MkDocs + */ + public static final PythonDependency MKDOCS_MATERIAL = new PythonDependency( + "mkdocs-material", + "~=9.6.22", Type.DOCS_DEPENDENCY, false); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java index 4aae8893f..99255dae5 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java @@ -333,10 +333,11 @@ private void generateConfig(GenerationContext context, PythonWriter writer) { var finalProperties = List.copyOf(properties); writer.pushState(new ConfigSection(finalProperties)); writer.addStdlibImport("dataclasses", "dataclass"); + var clientSymbol = context.symbolProvider().toSymbol(service); writer.write(""" @dataclass(init=False) class $L: - \"""Configuration for $L.\""" + \"""Configuration settings for $L.\""" ${C|} @@ -345,17 +346,12 @@ def __init__( *, ${C|} ): - \"""Constructor. - - ${C|} - \""" ${C|} """, configSymbol.getName(), - context.settings().service().getName(), + clientSymbol.getName(), writer.consumer(w -> writePropertyDeclarations(w, finalProperties)), writer.consumer(w -> writeInitParams(w, finalProperties)), - writer.consumer(w -> documentProperties(w, finalProperties)), writer.consumer(w -> initializeProperties(w, finalProperties))); writer.popState(); } @@ -366,6 +362,8 @@ private void writePropertyDeclarations(PythonWriter writer, Collection pro } } - private void documentProperties(PythonWriter writer, Collection properties) { - var iter = properties.iterator(); - while (iter.hasNext()) { - var property = iter.next(); - var docs = writer.formatDocs(String.format(":param %s: %s", property.name(), property.documentation())); - - if (iter.hasNext()) { - docs += "\n"; - } - - writer.write(docs); - } - } - private void initializeProperties(PythonWriter writer, Collection properties) { for (ConfigProperty property : properties) { property.initialize(writer); @@ -417,7 +401,9 @@ def set_auth_scheme(self, scheme: AuthScheme[Any, Any, Any, Any]) -> None: Using this method ensures the correct key is used. - :param scheme: The auth scheme to add. + Args: + scheme: + The auth scheme to add. \""" self.auth_schemes[scheme.scheme_id] = scheme """); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java index 8f0d2f804..f5c68e752 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java @@ -10,6 +10,7 @@ import software.amazon.smithy.model.traits.EnumValueTrait; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.SymbolProperties; +import software.amazon.smithy.python.codegen.sections.EnumSection; import software.amazon.smithy.utils.SmithyInternalApi; /** @@ -30,18 +31,21 @@ public void run() { var enumSymbol = context.symbolProvider().toSymbol(shape).expectProperty(SymbolProperties.ENUM_SYMBOL); context.writerDelegator().useShapeWriter(shape, writer -> { writer.addStdlibImport("enum", "StrEnum"); + writer.pushState(new EnumSection(shape, enumSymbol)); writer.openBlock("class $L(StrEnum):", "", enumSymbol.getName(), () -> { shape.getTrait(DocumentationTrait.class).ifPresent(trait -> { - writer.writeDocs(writer.formatDocs(trait.getValue())); + writer.writeDocs(trait.getValue(), context); }); for (MemberShape member : shape.members()) { var name = context.symbolProvider().toMemberName(member); var value = member.expectTrait(EnumValueTrait.class).expectStringValue(); writer.write("$L = $S", name, value); - member.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeDocs(trait.getValue())); + member.getTrait(DocumentationTrait.class) + .ifPresent(trait -> writer.writeDocs(trait.getValue(), context)); } }); + writer.popState(); }); } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java index 7fbb6bb5d..602002a17 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java @@ -11,6 +11,7 @@ import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; import software.amazon.smithy.python.codegen.SymbolProperties; +import software.amazon.smithy.python.codegen.sections.IntEnumSection; import software.amazon.smithy.utils.SmithyInternalApi; @SmithyInternalApi @@ -25,11 +26,13 @@ public IntEnumGenerator(GenerateIntEnumDirective { writer.addStdlibImport("enum", "IntEnum"); + writer.pushState(new IntEnumSection(intEnumShape, enumSymbol)); writer.openBlock("class $L(IntEnum):", "", enumSymbol.getName(), () -> { directive.shape().getTrait(DocumentationTrait.class).ifPresent(trait -> { - writer.writeDocs(writer.formatDocs(trait.getValue())); + writer.writeDocs(trait.getValue(), directive.context()); }); for (MemberShape member : directive.shape().members()) { @@ -39,6 +42,7 @@ public void run() { writer.write("$L = $L\n", name, value); } }); + writer.popState(); }); } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java index 343fccfd0..b371d53bc 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java @@ -40,7 +40,7 @@ public static void generateSetup( PythonSettings settings, GenerationContext context ) { - writeDocsSkeleton(settings, context); + setupDocs(settings, context); var dependencies = gatherDependencies(context.writerDelegator().getDependencies().stream()); writePyproject(settings, context.writerDelegator(), dependencies); writeReadme(settings, context); @@ -171,10 +171,11 @@ private static void writePyproject( requires = ["hatchling"] build-backend = "hatchling.build" - [tool.hatch.build.targets.bdist] + [tool.hatch.build] exclude = [ "tests", "docs", + "mkdocs.yml", ] [tool.pyright] @@ -269,193 +270,128 @@ private static void writeReadme( ### Documentation $L - """, documentation); + """, writer.formatDocs(documentation, context)); }); writer.popState(); }); } /** - * Write the files required for sphinx doc generation + * Setup files and dependencies for MkDocs documentation generation. + * + * MkDocs documentation is only generated for AWS services. Generic clients + * receive docstrings but are free to choose their own documentation approach. */ - private static void writeDocsSkeleton( + private static void setupDocs( PythonSettings settings, GenerationContext context ) { - //TODO Add a configurable flag to disable the generation of the sphinx files - //TODO Add a configuration that will allow users to select a sphinx theme + // Skip generic services + if (!CodegenUtils.isAwsService(context)) { + return; + } + context.writerDelegator().useFileWriter("pyproject.toml", "", writer -> { - writer.addDependency(SmithyPythonDependency.SPHINX); - writer.addDependency(SmithyPythonDependency.SPHINX_PYDATA_THEME); + writer.addDependency(SmithyPythonDependency.MKDOCS); + writer.addDependency(SmithyPythonDependency.MKDOCSTRINGS); + writer.addDependency(SmithyPythonDependency.MKDOCS_MATERIAL); }); + var service = context.model().expectShape(settings.service()); - String projectName = service.getTrait(TitleTrait.class) - .map(StringTrait::getValue) - .orElseGet(() -> service.getTrait(ServiceTrait.class) - .map(ServiceTrait::getSdkId) + String projectName = service.getTrait(ServiceTrait.class) + .map(ServiceTrait::getSdkId) + .orElseGet(() -> service.getTrait(TitleTrait.class) + .map(StringTrait::getValue) .orElse(context.settings().service().getName())); - writeConf(settings, context, projectName); - writeIndexes(context, projectName); + writeMkDocsConfig(context, projectName); writeDocsReadme(context); - writeMakeBat(context); - writeMakeFile(context); } /** - * Write a conf.py file. - * A conf.py file is a configuration file used by Sphinx, a documentation - * generation tool for Python projects. This file contains settings and - * configurations that control the behavior and appearance of the generated - * documentation. + * Write mkdocs.yml configuration file. + * This file configures MkDocs, a static site generator for project documentation. */ - private static void writeConf( - PythonSettings settings, + private static void writeMkDocsConfig( GenerationContext context, String projectName ) { - String version = settings.moduleVersion(); - context.writerDelegator().useFileWriter("docs/conf.py", "", writer -> { - writer.write(""" - import os - import sys - sys.path.insert(0, os.path.abspath('..')) - - project = '$L' - author = 'Amazon Web Services' - release = '$L' - - extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', - ] - - templates_path = ['_templates'] - exclude_patterns = [] - - autodoc_default_options = { - 'exclude-members': 'deserialize,deserialize_kwargs,serialize,serialize_members' - } - - html_theme = 'pydata_sphinx_theme' - html_theme_options = { - "logo": { - "text": "$L", - } - } - - autodoc_typehints = 'description' - """, projectName, version, projectName); - }); - } - - /** - * Write a make.bat file. - * A make.bat file is a batch script used on Windows to build Sphinx documentation. - * This script sets up the environment and runs the Sphinx build commands. - * - * @param context The generation context containing the writer delegator. - */ - private static void writeMakeBat( - GenerationContext context - ) { - context.writerDelegator().useFileWriter("docs/make.bat", "", writer -> { - writer.write(""" - @ECHO OFF - - pushd %~dp0 - - REM Command file for Sphinx documentation - - if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build - ) - set BUILDDIR=build - set SERVICESDIR=source/reference/services - set SPHINXOPTS=-j auto - set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . - - if "%1" == "" goto help - - if "%1" == "clean" ( - rmdir /S /Q %BUILDDIR% - goto end - ) - - if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - echo. - echo "Build finished. The HTML pages are in %BUILDDIR%/html." - goto end - ) - - :help - %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - - :end - popd - """); - }); - } - - /** - * Write a Makefile. - * A Makefile is used on Unix-based systems to build Sphinx documentation. - * This file contains rules for cleaning the build directory and generating HTML documentation. - * - * @param context The generation context containing the writer delegator. - */ - private static void writeMakeFile( - GenerationContext context - ) { - context.writerDelegator().useFileWriter("docs/Makefile", "", writer -> { + context.writerDelegator().useFileWriter("mkdocs.yml", "", writer -> { writer.write(""" - SPHINXBUILD = sphinx-build - BUILDDIR = build - SERVICESDIR = source/reference/services - SPHINXOPTS = -j auto - ALLSPHINXOPTS = -d $$(BUILDDIR)/doctrees $$(SPHINXOPTS) . - - clean: - \t-rm -rf $$(BUILDDIR)/* - - html: - \t$$(SPHINXBUILD) -b html $$(ALLSPHINXOPTS) $$(BUILDDIR)/html - \t@echo - \t@echo "Build finished. The HTML pages are in $$(BUILDDIR)/html." - """); + site_name: AWS $1L + site_description: Documentation for $1L Client + + copyright: Copyright © 2025, Amazon Web Services, Inc + repo_name: awslabs/aws-sdk-python + repo_url: https://github.com/awslabs/aws-sdk-python + + theme: + name: material + palette: + - scheme: default + primary: white + accent: light blue + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + primary: black + accent: light blue + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - navigation.top + - search.suggest + - search.highlight + - content.code.copy + + plugins: + - search + - mkdocstrings: + handlers: + python: + options: + show_source: true + show_signature: true + show_signature_annotations: true + show_root_heading: true + show_root_full_path: false + show_object_full_path: false + show_symbol_type_heading: true + show_symbol_type_toc: true + show_category_heading: true + group_by_category: true + separate_signature: true + signature_crossrefs: true + filters: + - "!^_" + - "!^deserialize" + - "!^serialize" + + markdown_extensions: + - pymdownx.highlight + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - admonition + - def_list + - toc: + permalink: true + toc_depth: 3 + + nav: + - $1L: index.md + + extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/awslabs/aws-sdk-python + extra_css: + - stylesheets/extra.css + """, projectName); }); } - /** - * Write the main index files for the documentation. - * This method creates the main index.rst file and additional index files for - * the client and models sections. - * - * @param context The generation context containing the writer delegator. - */ - private static void writeIndexes(GenerationContext context, String projectName) { - // Write the main index file for the documentation - context.writerDelegator().useFileWriter("docs/index.rst", "", writer -> { - writer.write(""" - $L - $L - - .. toctree:: - :maxdepth: 2 - :titlesonly: - :glob: - - */index - """, projectName, "=".repeat(projectName.length())); - }); - - // Write the index file for the client section - writeIndexFile(context, "docs/client/index.rst", "Client"); - - // Write the index file for the models section - writeIndexFile(context, "docs/models/index.rst", "Models"); - } - /** * Write the readme in the docs folder describing instructions for generation * @@ -468,38 +404,20 @@ private static void writeDocsReadme( writer.write(""" ## Generating Documentation - Sphinx is used for documentation. You can generate HTML locally with the + Material for MkDocs is used for documentation. You can generate HTML locally with the following: - ``` - $$ uv pip install --group docs . - $$ cd docs - $$ make html - ``` - """); - }); - } + ```bash + # Install documentation dependencies + uv pip install --group docs - /** - * Helper method to write an index file with the given title. - * This method creates an index file at the specified file path with the provided title. - * - * @param context The generation context. - * @param filePath The file path of the index file. - * @param title The title of the index file. - */ - private static void writeIndexFile(GenerationContext context, String filePath, String title) { - context.writerDelegator().useFileWriter(filePath, "", writer -> { - writer.write(""" - $L - ======= - .. toctree:: - :maxdepth: 1 - :titlesonly: - :glob: - - * - """, title); + # Serve documentation locally + mkdocs serve + + # OR build static HTML documentation + mkdocs build + ``` + """); }); } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java index 1cfa9c2cf..9c74c2f93 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java @@ -20,11 +20,9 @@ import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.StructureShape; -import software.amazon.smithy.model.traits.ClientOptionalTrait; import software.amazon.smithy.model.traits.DefaultTrait; import software.amazon.smithy.model.traits.DocumentationTrait; import software.amazon.smithy.model.traits.ErrorTrait; -import software.amazon.smithy.model.traits.RequiredTrait; import software.amazon.smithy.model.traits.RetryableTrait; import software.amazon.smithy.model.traits.SensitiveTrait; import software.amazon.smithy.model.traits.StreamingTrait; @@ -112,7 +110,7 @@ class $L: """, symbol.getName(), - writer.consumer(this::writeClassDocs), + writer.consumer(w -> writeClassDocs()), writer.consumer(w -> writeProperties()), writer.consumer(w -> generateSerializeMethod()), writer.consumer(w -> generateDeserializeMethod())); @@ -160,7 +158,7 @@ class $1L($2T): symbol.getName(), baseError, fault, - writer.consumer(this::writeClassDocs), + writer.consumer(w -> writeClassDocs()), writer.consumer(w -> writeProperties()), writer.consumer(w -> generateSerializeMethod()), writer.consumer(w -> generateDeserializeMethod())); @@ -168,6 +166,13 @@ class $1L($2T): } + private void writeClassDocs() { + var docs = shape.getTrait(DocumentationTrait.class) + .map(DocumentationTrait::getValue) + .orElse("Dataclass for " + shape.getId().getName() + " structure."); + writer.writeDocs(docs, context); + } + private void writeProperties() { for (MemberShape member : requiredMembers) { writer.pushState(); @@ -179,21 +184,17 @@ private void writeProperties() { } else { writer.putContext("sensitive", false); } - var docs = member.getMemberTrait(model, DocumentationTrait.class) - .map(DocumentationTrait::getValue) - .map(writer::formatDocs) - .orElse(null); - writer.putContext("docs", docs); var memberName = symbolProvider.toMemberName(member); writer.putContext("quote", recursiveShapes.contains(target) ? "'" : ""); writer.write(""" $L: ${quote:L}$T${quote:L}\ ${?sensitive} = field(repr=False)${/sensitive} - ${?docs}""\"${docs:L}""\"${/docs} + $C """, memberName, - symbolProvider.toSymbol(member)); + symbolProvider.toSymbol(member), + writer.consumer(w -> writeMemberDocs(member))); writer.popState(); } @@ -227,11 +228,6 @@ private void writeProperties() { writer.putContext("defaultKey", defaultKey); writer.putContext("defaultValue", defaultValue); writer.putContext("useField", requiresField); - var docs = member.getMemberTrait(model, DocumentationTrait.class) - .map(DocumentationTrait::getValue) - .map(writer::formatDocs) - .orElse(null); - writer.putContext("docs", docs); writer.putContext("quote", recursiveShapes.contains(target) ? "'" : ""); @@ -242,14 +238,18 @@ private void writeProperties() { ${?useField}\ field(${?sensitive}repr=False, ${/sensitive}${defaultKey:L}=${defaultValue:L})\ ${/useField} - ${?docs}""\"${docs:L}""\"${/docs}""", memberName, symbolProvider.toSymbol(member)); + $C + """, + memberName, + symbolProvider.toSymbol(member), + writer.consumer(w -> writeMemberDocs(member))); writer.popState(); } } - private void writeClassDocs(PythonWriter writer) { - shape.getTrait(DocumentationTrait.class).ifPresent(trait -> { - writer.writeDocs(writer.formatDocs(trait.getValue()).trim()); + private void writeMemberDocs(MemberShape member) { + member.getMemberTrait(model, DocumentationTrait.class).ifPresent(trait -> { + writer.writeDocs(trait.getValue(), context); }); } @@ -303,34 +303,6 @@ private String getDefaultValue(PythonWriter writer, MemberShape member) { }; } - private boolean hasDocs() { - if (shape.hasTrait(DocumentationTrait.class)) { - return true; - } - for (MemberShape member : shape.members()) { - if (member.getMemberTrait(model, DocumentationTrait.class).isPresent()) { - return true; - } - } - return false; - } - - private void writeMemberDocs(MemberShape member) { - member.getMemberTrait(model, DocumentationTrait.class).ifPresent(trait -> { - String descriptionPrefix = ""; - if (member.hasTrait(RequiredTrait.class) && !member.hasTrait(ClientOptionalTrait.class)) { - descriptionPrefix = "**[Required]** - "; - } - - String memberName = symbolProvider.toMemberName(member); - String docs = writer.formatDocs(String.format(":param %s: %s%s", - memberName, - descriptionPrefix, - trait.getValue())); - writer.write(docs); - }); - } - private void generateSerializeMethod() { writer.pushState(); writer.addImport("smithy_core.serializers", "ShapeSerializer"); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java index c24c6590f..908026142 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java @@ -15,7 +15,6 @@ import software.amazon.smithy.model.traits.StringTrait; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.SymbolProperties; -import software.amazon.smithy.python.codegen.sections.UnionMemberSection; import software.amazon.smithy.python.codegen.sections.UnionSection; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; @@ -64,7 +63,6 @@ public void run() { var target = model.expectShape(member.getTarget()); var targetSymbol = symbolProvider.toSymbol(target); - writer.pushState(new UnionMemberSection(memberSymbol)); writer.write(""" @dataclass class $1L: @@ -86,7 +84,7 @@ def deserialize(cls, deserializer: ShapeDeserializer) -> Self: memberSymbol.getName(), writer.consumer(w -> member.getMemberTrait(model, DocumentationTrait.class) .map(StringTrait::getValue) - .ifPresent(w::writeDocs)), + .ifPresent(docs -> w.writeDocs(docs, context))), targetSymbol, schemaSymbol, writer.consumer(w -> target.accept( @@ -95,7 +93,6 @@ def deserialize(cls, deserializer: ShapeDeserializer) -> Self: new MemberDeserializerGenerator(context, w, member, "deserializer"))) ); - writer.popState(); } // Note that the unknown variant doesn't implement __eq__. This is because @@ -103,7 +100,6 @@ def deserialize(cls, deserializer: ShapeDeserializer) -> Self: // Since the underlying value is unknown and un-comparable, that is the only // realistic implementation. var unknownSymbol = symbolProvider.toSymbol(shape).expectProperty(SymbolProperties.UNION_UNKNOWN); - writer.pushState(new UnionMemberSection(unknownSymbol)); writer.addImport("smithy_core.exceptions", "SerializationError"); writer.write(""" @dataclass @@ -130,13 +126,12 @@ raise NotImplementedError() """, unknownSymbol.getName()); memberNames.add(unknownSymbol.getName()); - writer.popState(); writer.pushState(new UnionSection(shape, parentName, memberNames)); // We need to use the old union syntax until we either migrate away from // Sphinx or Sphinx fixes the issue upstream: https://github.com/sphinx-doc/sphinx/issues/10785 - writer.write("$L = Union[$L]\n", parentName, String.join(" | ", memberNames)); - shape.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeDocs(trait.getValue())); + writer.write("$L = Union[$L]", parentName, String.join(" | ", memberNames)); + shape.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeDocs(trait.getValue(), context)); writer.popState(); generateDeserializer(); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionMemberSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/EnumSection.java similarity index 65% rename from codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionMemberSection.java rename to codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/EnumSection.java index bd6d999e6..0325a37a3 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionMemberSection.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/EnumSection.java @@ -5,11 +5,12 @@ package software.amazon.smithy.python.codegen.sections; import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.shapes.EnumShape; import software.amazon.smithy.utils.CodeSection; import software.amazon.smithy.utils.SmithyInternalApi; /** - * A section that controls writing a union member. + * A section that controls writing an enum. */ @SmithyInternalApi -public record UnionMemberSection(Symbol memberSymbol) implements CodeSection {} +public record EnumSection(EnumShape enumShape, Symbol enumSymbol) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/IntEnumSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/IntEnumSection.java new file mode 100644 index 000000000..7f28e7b9c --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/IntEnumSection.java @@ -0,0 +1,16 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.model.shapes.IntEnumShape; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * A section that controls writing an integer enum. + */ +@SmithyInternalApi +public record IntEnumSection(IntEnumShape intEnumShape, Symbol enumSymbol) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownConverter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownConverter.java new file mode 100644 index 000000000..68b2ced97 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownConverter.java @@ -0,0 +1,169 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.writer; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Converts CommonMark/HTML documentation to Markdown for Python docstrings. + * + * This converter uses the pandoc CLI tool to convert documentation from CommonMark/HTML + * format to Markdown suitable for Python docstrings with Google-style formatting. + * + * Pandoc must be installed and available. + */ +@SmithyInternalApi +public final class MarkdownConverter { + + private static final int TIMEOUT_SECONDS = 10; + + // Private constructor to prevent instantiation + private MarkdownConverter() {} + + /** + * Converts HTML or CommonMark strings to Markdown format using pandoc. + * + * For AWS services, documentation is in HTML format with raw HTML tags. + * For generic services, documentation is in CommonMark format which can + * include embedded HTML. + * + * @param input The input string (HTML or CommonMark) + * @param context The generation context to determine service type + * @return Markdown formatted string + */ + public static String convert(String input, GenerationContext context) { + if (input == null || input.isEmpty()) { + return ""; + } + + try { + if (!CodegenUtils.isAwsService(context)) { + // Commonmark may include embedded HTML so we first normalize the input to HTML format + input = convertWithPandoc(input, "commonmark", "html"); + } + + // The "html+raw_html" format preserves unrecognized html tags (e.g. , ) + // in Markdown output. We convert these tags to admonitions in postProcressPandocOutput() + String output = convertWithPandoc(input, "html+raw_html", "markdown"); + + return postProcessPandocOutput(output); + } catch (IOException | InterruptedException e) { + throw new CodegenException("Failed to convert documentation using pandoc: " + e.getMessage(), e); + } + } + + /** + * Calls pandoc CLI to convert documentation. + * + * @param input The input string + * @param fromFormat The input format (e.g., "html+raw_html" or "commonmark") + * @param toFormat The output format (e.g., "markdown") + * @return Converted Markdown string + * @throws IOException if process I/O fails + * @throws InterruptedException if process is interrupted + */ + private static String convertWithPandoc(String input, String fromFormat, String toFormat) + throws IOException, InterruptedException { + ProcessBuilder processBuilder = new ProcessBuilder( + "pandoc", + "--from=" + fromFormat, + "--to=" + toFormat, + "--wrap=auto", + "--columns=72"); + processBuilder.redirectErrorStream(true); + + Process process = processBuilder.start(); + + // Write input to pandoc's stdin + try (var outputStream = process.getOutputStream()) { + outputStream.write(input.getBytes(StandardCharsets.UTF_8)); + outputStream.flush(); + } + + // Read output from pandoc's stdout + StringBuilder output = new StringBuilder(); + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + output.append(line).append("\n"); + } + } + + // Wait for process to complete + boolean finished = process.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS); + if (!finished) { + process.destroyForcibly(); + throw new CodegenException("Pandoc process timed out after " + TIMEOUT_SECONDS + " seconds"); + } + + int exitCode = process.exitValue(); + if (exitCode != 0) { + throw new CodegenException( + "Pandoc failed with exit code " + exitCode + ": " + output.toString().trim()); + } + + return output.toString(); + } + + private static String postProcessPandocOutput(String output) { + // Remove empty lines at the start and end + output = output.trim(); + + // Remove unnecessary backslash escapes that pandoc adds for markdown + // These characters don't need escaping in Python docstrings + // Handles: [ ] ' { } ( ) < > ` @ _ * | ! ~ $ + output = output.replaceAll("\\\\([\\[\\]'{}()<>`@_*|!~$])", "$1"); + + // Replace and tags with admonitions for mkdocstrings + output = replaceAdmonitionTags(output, "note", "Note"); + output = replaceAdmonitionTags(output, "important", "Warning"); + + // Escape Smithy format specifiers + return output.replace("$", "$$"); + } + + /** + * Replaces admonition tags (e.g. note, important) with Google-style format. + * + * @param text The text to process + * @param tagName The tag name to replace (e.g., "note", "important") + * @param label The label to use (e.g., "Note", "Warning") + * @return Text with replaced admonitions + */ + private static String replaceAdmonitionTags(String text, String tagName, String label) { + // Match content across multiple lines + Pattern pattern = Pattern.compile("<" + tagName + ">\\s*([\\s\\S]*?)\\s*"); + Matcher matcher = pattern.matcher(text); + StringBuffer result = new StringBuffer(); + + while (matcher.find()) { + // Extract the content between tags + String content = matcher.group(1).trim(); + + // Indent each line with 4 spaces + String[] lines = content.split("\n"); + StringBuilder indented = new StringBuilder(label + ":\n"); + for (String line : lines) { + indented.append(" ").append(line.trim()).append("\n"); + } + + matcher.appendReplacement(result, Matcher.quoteReplacement(indented.toString().trim())); + } + matcher.appendTail(result); + + return result.toString(); + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java deleted file mode 100644 index 883e9b132..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.writer; - -import static org.jsoup.nodes.Document.OutputSettings.Syntax.html; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.commonmark.node.BlockQuote; -import org.commonmark.node.FencedCodeBlock; -import org.commonmark.node.Heading; -import org.commonmark.node.HtmlBlock; -import org.commonmark.node.ListBlock; -import org.commonmark.node.ThematicBreak; -import org.commonmark.parser.Parser; -import org.commonmark.renderer.html.HtmlRenderer; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.nodes.Node; -import org.jsoup.nodes.TextNode; -import org.jsoup.select.NodeVisitor; -import software.amazon.smithy.utils.SetUtils; -import software.amazon.smithy.utils.SimpleCodeWriter; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * Add a runtime plugin to convert the HTML docs that are provided by services into RST - */ -@SmithyInternalApi -public class MarkdownToRstDocConverter { - private static final Parser MARKDOWN_PARSER = Parser.builder() - .enabledBlockTypes(SetUtils.of( - Heading.class, - HtmlBlock.class, - ThematicBreak.class, - FencedCodeBlock.class, - BlockQuote.class, - ListBlock.class)) - .build(); - - // Singleton instance - private static final MarkdownToRstDocConverter DOC_CONVERTER = new MarkdownToRstDocConverter(); - - // Private constructor to prevent instantiation - private MarkdownToRstDocConverter() { - // Constructor - } - - public static MarkdownToRstDocConverter getInstance() { - return DOC_CONVERTER; - } - - public String convertCommonmarkToRst(String commonmark) { - String html = HtmlRenderer.builder().escapeHtml(false).build().render(MARKDOWN_PARSER.parse(commonmark)); - //Replace the outer HTML paragraph tag with a div tag - Pattern pattern = Pattern.compile("^

(.*)

$", Pattern.DOTALL); - Matcher matcher = pattern.matcher(html); - html = matcher.replaceAll("
$1
"); - Document document = Jsoup.parse(html); - RstNodeVisitor visitor = new RstNodeVisitor(); - document.body().traverse(visitor); - return "\n" + visitor; - } - - private static class RstNodeVisitor implements NodeVisitor { - SimpleCodeWriter writer = new SimpleCodeWriter(); - private boolean inList = false; - private int listDepth = 0; - - @Override - public void head(Node node, int depth) { - if (node instanceof TextNode) { - TextNode textNode = (TextNode) node; - String text = textNode.text(); - if (!text.trim().isEmpty()) { - if (text.startsWith(":param ")) { - int secondColonIndex = text.indexOf(':', 1); - writer.write("$L", text.substring(0, secondColonIndex + 1)); - //TODO right now the code generator gives us a mixture of - // RST and HTML (for instance :param xyz:

docs - //

). Since we standardize to html above, that

tag - // starts a newline. We account for that with this if/else - // statement, but we should refactor this in the future to - // have a more elegant codepath. - if (secondColonIndex + 1 == text.strip().length()) { - writer.indent(); - writer.ensureNewline(); - } else { - writer.ensureNewline(); - writer.indent(); - writer.write("$L", text.substring(secondColonIndex + 1)); - writer.dedent(); - } - } else { - writer.writeInline("$L", text); - } - // Account for services making a paragraph tag that's empty except - // for a newline - } else if (node.parent() != null && ((Element) node.parent()).tagName().equals("p")) { - writer.writeInline("$L", text.replaceAll("[ \\t]+", "")); - } - } else if (node instanceof Element) { - Element element = (Element) node; - switch (element.tagName()) { - case "a": - writer.writeInline("`"); - break; - case "b": - case "strong": - writer.writeInline("**"); - break; - case "i": - case "em": - writer.writeInline("*"); - break; - case "code": - writer.writeInline("``"); - break; - case "important": - writer.ensureNewline(); - writer.write(""); - writer.openBlock(".. important::"); - break; - case "note": - writer.ensureNewline(); - writer.write(""); - writer.openBlock(".. note::"); - break; - case "ul": - if (inList) { - writer.indent(); - } - inList = true; - listDepth++; - writer.ensureNewline(); - writer.write(""); - break; - case "li": - writer.writeInline("* "); - break; - case "h1": - writer.ensureNewline(); - break; - default: - break; - } - } - } - - @Override - public void tail(Node node, int depth) { - if (node instanceof Element) { - Element element = (Element) node; - switch (element.tagName()) { - case "a": - String href = element.attr("href"); - if (!href.isEmpty()) { - writer.writeInline(" <").writeInline("$L", href).writeInline(">`_"); - } else { - writer.writeInline("`"); - } - break; - case "b": - case "strong": - writer.writeInline("**"); - break; - case "i": - case "em": - writer.writeInline("*"); - break; - case "code": - writer.writeInline("``"); - break; - case "important": - case "note": - writer.closeBlock(""); - break; - case "p": - writer.ensureNewline(); - writer.write(""); - break; - case "ul": - listDepth--; - if (listDepth == 0) { - inList = false; - } else { - writer.dedent(); - } - writer.ensureNewline(); - break; - case "li": - writer.ensureNewline(); - break; - case "h1": - String title = element.text(); - writer.ensureNewline().writeInline("$L", "=".repeat(title.length())).ensureNewline(); - break; - default: - break; - } - } - } - - @Override - public String toString() { - return writer.toString(); - } - } -} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java index 6d3ecac94..978f4fd77 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java @@ -23,7 +23,7 @@ import software.amazon.smithy.model.node.NumberNode; import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.node.StringNode; -import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; import software.amazon.smithy.utils.SmithyUnstableApi; import software.amazon.smithy.utils.StringUtils; @@ -39,10 +39,9 @@ public final class PythonWriter extends SymbolWriter { private static final Logger LOGGER = Logger.getLogger(PythonWriter.class.getName()); - private static final MarkdownToRstDocConverter DOC_CONVERTER = MarkdownToRstDocConverter.getInstance(); private final String fullPackageName; - private final String commentStart; + private final boolean addCodegenWarningHeader; private boolean addLogger = false; /** @@ -52,7 +51,7 @@ public final class PythonWriter extends SymbolWriter write(formatDocs(docs))); + public PythonWriter writeDocs(String docs, GenerationContext context) { + String formatted = formatDocs(docs, context); + // For single-line docstrings, keep the closing quotes on the same line + if (!formatted.contains("\n")) { + writeDocs(() -> writeInline(formatted)); + } else { + writeDocs(() -> write(formatted)); + } return this; } - private static final int MAX_LINE_LENGTH = CodegenUtils.MAX_PREFERRED_LINE_LENGTH - 8; - /** - * Formats a given Commonmark string and wraps it for use in a doc - * comment. + * Formats documentation from CommonMark or HTML to Google-style Markdown for Python docstrings. + * + *

For AWS services, expects HTML input. For generic clients, expects CommonMark input. * * @param docs Documentation to format. + * @param context The generation context used to determine service type and formatting. * @return Formatted documentation. */ - public String formatDocs(String docs) { - String rstDocs = DOC_CONVERTER.convertCommonmarkToRst(docs); - return wrapRST(rstDocs).toString().replace("$", "$$"); - } - - public static String wrapRST(String text) { - StringBuilder wrappedText = new StringBuilder(); - String[] lines = text.split("\n"); - for (String line : lines) { - wrapLine(line, wrappedText); - } - return wrappedText.toString(); - } - - private static void wrapLine(String line, StringBuilder wrappedText) { - int indent = getIndentationLevel(line); - String indentStr = " ".repeat(indent); - line = line.trim(); - - while (line.length() > MAX_LINE_LENGTH) { - int wrapAt = findWrapPosition(line, MAX_LINE_LENGTH); - wrappedText.append(indentStr).append(line, 0, wrapAt).append("\n"); - if (line.startsWith("* ")) { - indentStr += " "; - } - line = line.substring(wrapAt).trim(); - if (line.isEmpty()) { - return; - } - } - wrappedText.append(indentStr).append(line).append("\n"); - } - - private static int findWrapPosition(String line, int maxLineLength) { - // Find the last space before maxLineLength - int wrapAt = line.lastIndexOf(' ', maxLineLength); - if (wrapAt == -1) { - // If no space found, don't wrap - wrapAt = line.length(); - } else { - // Ensure we don't break a link - //TODO account for earlier backticks on the same line as a link - int linkStart = line.lastIndexOf("`", wrapAt); - int linkEnd = line.indexOf("`_", wrapAt); - if (linkStart != -1 && (linkEnd != -1 && linkEnd > linkStart)) { - linkEnd = line.indexOf("`_", linkStart); - if (linkEnd != -1) { - wrapAt = linkEnd + 2; - } else { - // No matching `_` found, keep the original wrap position - wrapAt = line.lastIndexOf(' ', maxLineLength); - if (wrapAt == -1) { - wrapAt = maxLineLength; - } - } - } - } - // Include trailing punctuation before a space in the previous line - int nextSpace = line.indexOf(' ', wrapAt); - if (nextSpace != -1) { - int i = wrapAt; - while (i < nextSpace && !Character.isLetterOrDigit(line.charAt(i))) { - i++; - } - if (i == nextSpace) { - wrapAt = nextSpace; - } - } - return wrapAt; - } - - private static int getIndentationLevel(String line) { - int indent = 0; - while (indent < line.length() && Character.isWhitespace(line.charAt(indent))) { - indent++; - } - return indent; + public String formatDocs(String docs, GenerationContext context) { + return MarkdownConverter.convert(docs, context); } /** @@ -253,7 +173,7 @@ public PythonWriter openComment(Runnable runnable) { * @return Returns the writer. */ public PythonWriter writeComment(String comment) { - return openComment(() -> write(formatDocs(comment.replace("\n", " ")))); + return openComment(() -> write(comment.replace("\n", " "))); } /** @@ -373,8 +293,8 @@ public String toString() { } contents += super.toString(); - if (!commentStart.equals("")) { - String header = String.format("%s Code generated by smithy-python-codegen DO NOT EDIT.%n%n", commentStart); + if (addCodegenWarningHeader) { + String header = "# Code generated by smithy-python-codegen DO NOT EDIT.\n\n"; contents = header + contents; } return contents; diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/CodegenUtilsTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/CodegenUtilsTest.java new file mode 100644 index 000000000..df3848d02 --- /dev/null +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/CodegenUtilsTest.java @@ -0,0 +1,60 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; + +public class CodegenUtilsTest { + + @Test + public void testIsAwsServiceWithAwsServiceTrait() { + // Create a mock context with an AWS service + GenerationContext context = createMockContext(true); + + // Should return true for AWS services + assertTrue(CodegenUtils.isAwsService(context)); + } + + @Test + public void testIsAwsServiceWithoutAwsServiceTrait() { + // Create a mock context with a non-AWS service + GenerationContext context = createMockContext(false); + + // Should return false for non-AWS services + assertFalse(CodegenUtils.isAwsService(context)); + } + + /** + * Helper method to create a mock GenerationContext with a service shape. + * + * @param hasAwsServiceTrait whether the service has the AWS service trait + * @return a mocked GenerationContext + */ + private GenerationContext createMockContext(boolean hasAwsServiceTrait) { + GenerationContext context = mock(GenerationContext.class); + Model model = mock(Model.class); + PythonSettings settings = mock(PythonSettings.class); + + ShapeId serviceId = ShapeId.from("test.service#TestService"); + ServiceShape serviceShape = mock(ServiceShape.class); + + when(context.model()).thenReturn(model); + when(context.settings()).thenReturn(settings); + when(settings.service()).thenReturn(serviceId); + when(model.expectShape(serviceId)).thenReturn(serviceShape); + when(serviceShape.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class)) + .thenReturn(hasAwsServiceTrait); + + return context; + } +} diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/writer/MarkdownConverterTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/writer/MarkdownConverterTest.java new file mode 100644 index 000000000..47c4b2708 --- /dev/null +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/writer/MarkdownConverterTest.java @@ -0,0 +1,222 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.writer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; + +public class MarkdownConverterTest { + + private GenerationContext createMockContext(boolean isAwsService) { + GenerationContext context = mock(GenerationContext.class); + Model model = mock(Model.class); + PythonSettings settings = mock(PythonSettings.class); + + ShapeId serviceId = ShapeId.from("test.service#TestService"); + ServiceShape serviceShape = mock(ServiceShape.class); + + when(context.model()).thenReturn(model); + when(context.settings()).thenReturn(settings); + when(settings.service()).thenReturn(serviceId); + when(model.expectShape(serviceId)).thenReturn(serviceShape); + + if (isAwsService) { + when(serviceShape.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class)).thenReturn(true); + } else { + when(serviceShape.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class)).thenReturn(false); + } + + return context; + } + + @Test + public void testConvertHtmlToMarkdownWithTitleAndParagraph() { + String html = "

Title

Paragraph

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should contain markdown heading and paragraph + String expected = """ + # Title + + Paragraph"""; + assertEquals(expected, result); + } + + @Test + public void testConvertHtmlToMarkdownWithList() { + String html = "
  • Item 1
  • Item 2
"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should contain markdown list + String expected = """ + - Item 1 + - Item 2"""; + assertEquals(expected, result); + } + + @Test + public void testConvertHtmlToMarkdownWithBoldTag() { + String html = "Bold text"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("**Bold text**", result); + } + + @Test + public void testConvertHtmlToMarkdownWithItalicTag() { + String html = "Italic text"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("*Italic text*", result); + } + + @Test + public void testConvertHtmlToMarkdownWithCodeTag() { + String html = "code snippet"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("`code snippet`", result); + } + + @Test + public void testConvertHtmlToMarkdownWithAnchorTag() { + String html = "Link"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("[Link](https://example.com)", result); + } + + @Test + public void testConvertHtmlToMarkdownWithNestedList() { + String html = "
  • Item 1
    • Subitem 1
  • Item 2
"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should contain nested markdown list with proper indentation + String expected = """ + - Item 1 + - Subitem 1 + - Item 2"""; + assertEquals(expected, result); + } + + @Test + public void testConvertHtmlToMarkdownWithFormatSpecifierCharacters() { + // Test that Smithy format specifier characters ($) are properly escaped + String html = "

Testing $placeholderOne and $placeholderTwo

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // $ should be escaped to $$ + assertEquals("Testing $$placeholderOne and $$placeholderTwo", result); + } + + @Test + public void testConvertCommonmarkWithEmbeddedHtmlForNonAwsService() { + // For non-AWS services, input is CommonMark which may include embedded HTML + // This tests the important path where commonmark -> html -> markdown conversion happens + String commonmarkWithHtml = "# Title\n\nParagraph with **bold** text and embedded HTML."; + String result = MarkdownConverter.convert(commonmarkWithHtml, createMockContext(false)); + // Should properly handle both markdown syntax and embedded HTML + String expected = """ + # Title + + Paragraph with **bold** text and *embedded HTML*."""; + assertEquals(expected, result); + } + + @Test + public void testConvertPureCommonmarkForNonAwsService() { + // For non-AWS services with pure CommonMark (no embedded HTML) + String commonmark = "# Title\n\nParagraph with **bold** and *italic* text.\n\n- List item 1\n- List item 2"; + String result = MarkdownConverter.convert(commonmark, createMockContext(false)); + // Should preserve the markdown structure (pandoc uses single space after dash) + String expected = """ + # Title + + Paragraph with **bold** and *italic* text. + + - List item 1 + - List item 2"""; + assertEquals(expected, result); + } + + @Test + public void testConvertRemovesUnnecessaryBackslashEscapes() { + // Pandoc adds escapes for these characters but they're not needed in Python docstrings + String html = "

Text with [brackets] and {braces} and (parens)

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should not have backslash escapes for these characters + assertEquals("Text with [brackets] and {braces} and (parens)", result.trim()); + } + + @Test + public void testConvertMixedElements() { + String html = "

Title

Paragraph

  • Item 1
  • Item 2
"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + String expected = "# Title\n\nParagraph\n\n- Item 1\n- Item 2"; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertNestedElements() { + String html = "

Title

Paragraph with bold text

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + String expected = """ + # Title + + Paragraph with **bold** text"""; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertMultilineText() { + // Create a note with content > 72 chars to trigger wrapping + String longLine = + "This is a very long line that exceeds seventy-two characters and should wrap into two lines."; + String html = "" + longLine + ""; + String result = MarkdownConverter.convert(html, createMockContext(true)); + String expected = """ + This is a very long line that exceeds seventy-two characters and should + wrap into two lines."""; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertHtmlToMarkdownWithNoteTag() { + String html = "Note text"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should convert to admonition format + String expected = """ + Note: + Note text"""; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertHtmlToMarkdownWithImportantTag() { + String html = "Important text"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should convert to warning admonition + String expected = """ + Warning: + Important text"""; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertMultilineAdmonitionTag() { + // Create a note with content > 72 chars to trigger wrapping + String longLine = + "This is a very long line that exceeds seventy-two characters and should wrap into two lines."; + String html = "" + longLine + ""; + String result = MarkdownConverter.convert(html, createMockContext(true)); + + // Expected: first line up to 72 chars, rest on second line, both indented + String expected = """ + Note: + This is a very long line that exceeds seventy-two characters and should + wrap into two lines."""; + assertEquals(expected, result.trim()); + } +} diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/writer/PythonWriterTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/writer/PythonWriterTest.java new file mode 100644 index 000000000..6138dbd94 --- /dev/null +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/writer/PythonWriterTest.java @@ -0,0 +1,96 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.writer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; + +public class PythonWriterTest { + + private PythonSettings createMockSettings() { + PythonSettings settings = mock(PythonSettings.class); + when(settings.moduleName()).thenReturn("testmodule"); + return settings; + } + + private GenerationContext createMockContext(boolean isAwsService) { + GenerationContext context = mock(GenerationContext.class); + Model model = mock(Model.class); + PythonSettings settings = createMockSettings(); + + ShapeId serviceId = ShapeId.from("test.service#TestService"); + ServiceShape serviceShape = mock(ServiceShape.class); + + when(context.model()).thenReturn(model); + when(context.settings()).thenReturn(settings); + when(settings.service()).thenReturn(serviceId); + when(model.expectShape(serviceId)).thenReturn(serviceShape); + + if (isAwsService) { + when(serviceShape.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class)).thenReturn(true); + } else { + when(serviceShape.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class)).thenReturn(false); + } + + return context; + } + + @Test + public void testWriteDocsShortStringOnSingleLine() { + // Short documentation should be on a single line + PythonWriter writer = new PythonWriter(createMockSettings(), "test.module", false); + String shortDocs = "

This is a short documentation string.

"; + + writer.writeDocs(shortDocs, createMockContext(true)); + String output = writer.toString(); + + String expected = "\"\"\"This is a short documentation string.\"\"\"\n"; + assertEquals(expected, output); + } + + @Test + public void testWriteDocsVeryLongStringWrappedCorrectly() { + // Very long documentation should be wrapped at 72 characters per line + PythonWriter writer = new PythonWriter(createMockSettings(), "test.module", false); + String veryLongDocs = "

This is an extremely long documentation string that definitely exceeds " + + "the 72 character limit set by pandoc and should be wrapped to multiple lines " + + "to ensure readability and proper formatting in the generated Python code.

"; + + writer.writeDocs(veryLongDocs, createMockContext(true)); + String output = writer.toString(); + + String expected = """ + ""\"This is an extremely long documentation string that definitely exceeds + the 72 character limit set by pandoc and should be wrapped to multiple + lines to ensure readability and proper formatting in the generated + Python code. + ""\" + """; + assertEquals(expected, output); + } + + @Test + public void testWriteDocsPreservesDollarSigns() { + // Documentation with $ should be preserved in final output + // (The intermediate format uses $$ for Smithy, but final output has $) + PythonWriter writer = new PythonWriter(createMockSettings(), "test.module", false); + String docsWithDollar = "

Use $variable in your code.

"; + + writer.writeDocs(docsWithDollar, createMockContext(true)); + String output = writer.toString(); + + String expected = "\"\"\"Use $variable in your code.\"\"\"\n"; + assertEquals(expected, output); + } + +} diff --git a/codegen/gradle/libs.versions.toml b/codegen/gradle/libs.versions.toml index e28c67c7b..940c1091e 100644 --- a/codegen/gradle/libs.versions.toml +++ b/codegen/gradle/libs.versions.toml @@ -1,5 +1,6 @@ [versions] junit5 = "6.0.0" +mockito = "5.20.0" smithy = "1.63.0" test-logger-plugin = "4.0.0" spotbugs = "6.0.22" @@ -23,6 +24,7 @@ junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.re junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit5" } junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit5" } junit-platform-launcher = { module = "org.junit.platform:junit-platform-launcher" } +mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } # plugin artifacts for buildsrc plugins test-logger-plugin = { module = "com.adarshr:gradle-test-logger-plugin", version.ref = "test-logger-plugin" }