diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java new file mode 100644 index 000000000..ec81cbdc7 --- /dev/null +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java @@ -0,0 +1,200 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.aws.codegen; + +import static software.amazon.smithy.python.codegen.SymbolProperties.OPERATION_METHOD; + +import java.util.List; +import software.amazon.smithy.model.traits.InputTrait; +import software.amazon.smithy.model.traits.OutputTrait; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.integrations.PythonIntegration; +import software.amazon.smithy.python.codegen.sections.*; +import software.amazon.smithy.python.codegen.writer.PythonWriter; +import software.amazon.smithy.utils.CodeInterceptor; +import software.amazon.smithy.utils.CodeSection; + +public class AwsRstDocFileGenerator implements PythonIntegration { + + @Override + public List> interceptors( + GenerationContext context + ) { + return List.of( + // We generate custom RST files for each member that we want to have + // its own page. This gives us much more fine-grained control of + // what gets generated than just using automodule or autoclass on + // the client would alone. + new OperationGenerationInterceptor(context), + new StructureGenerationInterceptor(context), + new ErrorGenerationInterceptor(context), + new UnionGenerationInterceptor(context), + new UnionMemberGenerationInterceptor(context)); + } + + /** + * Utility method to generate a header for documentation files. + * + * @param title The title of the section. + * @return A formatted header string. + */ + private static String generateHeader(String title) { + return String.format("%s%n%s%n%n", title, "=".repeat(title.length())); + } + + private static final class OperationGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + + public OperationGenerationInterceptor(GenerationContext context) { + this.context = context; + } + + @Override + public Class sectionType() { + return OperationSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, OperationSection section) { + var operation = section.operation(); + var operationSymbol = context.symbolProvider().toSymbol(operation).expectProperty(OPERATION_METHOD); + var input = context.model().expectShape(operation.getInputShape()); + var inputSymbol = context.symbolProvider().toSymbol(input); + var output = context.model().expectShape(operation.getOutputShape()); + var outputSymbol = context.symbolProvider().toSymbol(output); + + String operationName = operationSymbol.getName(); + String inputSymbolName = inputSymbol.toString(); + String outputSymbolName = outputSymbol.toString(); + String serviceName = context.symbolProvider().toSymbol(section.service()).getName(); + String docsFileName = String.format("docs/client/%s.rst", operationName); + String fullOperationReference = String.format("%s.client.%s.%s", + context.settings().moduleName(), + serviceName, + operationName); + + context.writerDelegator().useFileWriter(docsFileName, "", fileWriter -> { + fileWriter.write(generateHeader(operationName)); + fileWriter.write(".. automethod:: " + fullOperationReference + "\n\n"); + fileWriter.write(".. toctree::\n :hidden:\n :maxdepth: 2\n\n"); + fileWriter.write("=================\nInput:\n=================\n\n"); + fileWriter.write(".. autoclass:: " + inputSymbolName + "\n :members:\n"); + fileWriter.write("=================\nOutput:\n=================\n\n"); + fileWriter.write(".. autoclass:: " + outputSymbolName + "\n :members:\n"); + }); + } + } + + private static final class StructureGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + + public StructureGenerationInterceptor(GenerationContext context) { + this.context = context; + } + + @Override + public Class sectionType() { + return StructureSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, StructureSection section) { + var shape = section.structure(); + var symbol = context.symbolProvider().toSymbol(shape); + String docsFileName = String.format("docs/models/%s.rst", + symbol.getName()); + if (!shape.hasTrait(InputTrait.class) && !shape.hasTrait(OutputTrait.class)) { + context.writerDelegator().useFileWriter(docsFileName, "", writer -> { + writer.write(generateHeader(symbol.getName())); + writer.write(".. autoclass:: " + symbol.toString() + "\n :members:\n"); + }); + } + } + } + + private static final class ErrorGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + + public ErrorGenerationInterceptor(GenerationContext context) { + this.context = context; + } + + @Override + public Class sectionType() { + return ErrorSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, ErrorSection section) { + var symbol = section.errorSymbol(); + String docsFileName = String.format("docs/models/%s.rst", + symbol.getName()); + context.writerDelegator().useFileWriter(docsFileName, "", writer -> { + writer.write(generateHeader(symbol.getName())); + writer.write(".. autoexception:: " + symbol.toString() + "\n :members:\n :show-inheritance:\n"); + }); + } + } + + private static final class UnionGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + + public UnionGenerationInterceptor(GenerationContext context) { + this.context = context; + } + + @Override + public Class sectionType() { + return UnionSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, UnionSection section) { + String parentName = section.parentName(); + String docsFileName = String.format("docs/models/%s.rst", parentName); + context.writerDelegator().useFileWriter(docsFileName, "", writer -> { + writer.write(".. _" + parentName + ":\n\n"); + writer.write(generateHeader(parentName)); + writer.write( + ".. autodata:: " + context.symbolProvider().toSymbol(section.unionShape()).toString() + " \n"); + }); + } + } + + private static final class UnionMemberGenerationInterceptor + implements CodeInterceptor.Appender { + + private final GenerationContext context; + + public UnionMemberGenerationInterceptor(GenerationContext context) { + this.context = context; + } + + @Override + public Class sectionType() { + return UnionMemberSection.class; + } + + @Override + public void append(PythonWriter pythonWriter, UnionMemberSection section) { + var memberSymbol = section.memberSymbol(); + String symbolName = memberSymbol.getName(); + String docsFileName = String.format("docs/models/%s.rst", symbolName); + context.writerDelegator().useFileWriter(docsFileName, "", writer -> { + writer.write(".. _" + symbolName + ":\n\n"); + writer.write(generateHeader(symbolName)); + writer.write(".. autoclass:: " + memberSymbol.toString() + " \n"); + }); + } + } +} \ No newline at end of file diff --git a/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index a338df30c..2a8cfe6d6 100644 --- a/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -8,3 +8,4 @@ software.amazon.smithy.python.aws.codegen.AwsProtocolsIntegration software.amazon.smithy.python.aws.codegen.AwsServiceIdIntegration software.amazon.smithy.python.aws.codegen.AwsUserAgentIntegration software.amazon.smithy.python.aws.codegen.AwsStandardRegionalEndpointsIntegration +software.amazon.smithy.python.aws.codegen.AwsRstDocFileGenerator diff --git a/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java b/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java new file mode 100644 index 000000000..0f51a6433 --- /dev/null +++ b/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java @@ -0,0 +1,109 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.aws.codegen; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.smithy.python.codegen.writer.MarkdownToRstDocConverter; + +public class MarkdownToRstDocConverterTest { + + private MarkdownToRstDocConverter markdownToRstDocConverter; + + @BeforeEach + public void setUp() { + markdownToRstDocConverter = MarkdownToRstDocConverter.getInstance(); + } + + @Test + public void testConvertCommonmarkToRstWithTitleAndParagraph() { + String html = "

Title

Paragraph

"; + String expected = "\n\nTitle\n=====\nParagraph\n"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithImportantNote() { + String html = "Important note"; + String expected = "\n\n.. important::\n Important note\n"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithList() { + String html = "
  • Item 1
  • Item 2
"; + String expected = "\n\n* Item 1\n\n* Item 2\n\n"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithMixedElements() { + String html = "

Title

Paragraph

  • Item 1
  • Item 2
"; + String expected = "\n\nTitle\n=====\nParagraph\n\n* Item 1\n\n* Item 2\n\n"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithNestedElements() { + String html = "

Title

Paragraph with bold text

"; + String expected = "\n\nTitle\n=====\nParagraph with **bold** text\n"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithAnchorTag() { + String html = "Link"; + String expected = "\n`Link `_"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithBoldTag() { + String html = "Bold text"; + String expected = "\n**Bold text**"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithItalicTag() { + String html = "Italic text"; + String expected = "\n*Italic text*"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithCodeTag() { + String html = "code snippet"; + String expected = "\n``code snippet``"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithNoteTag() { + String html = "Note text"; + String expected = "\n\n.. note::\n Note text\n"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } + + @Test + public void testConvertCommonmarkToRstWithNestedList() { + String html = "
  • Item 1
    • Subitem 1
  • Item 2
"; + String expected = "\n\n* Item 1\n * Subitem 1\n\n* Item 2\n\n"; + String result = markdownToRstDocConverter.convertCommonmarkToRst(html); + assertEquals(expected, result); + } +} diff --git a/codegen/core/build.gradle.kts b/codegen/core/build.gradle.kts index bc8da8191..04f9064dd 100644 --- a/codegen/core/build.gradle.kts +++ b/codegen/core/build.gradle.kts @@ -15,4 +15,6 @@ dependencies { implementation(libs.smithy.protocol.test.traits) // We have this because we're using RestJson1 as a 'generic' protocol. implementation(libs.smithy.aws.traits) + implementation(libs.jsoup) + implementation(libs.commonmark) } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 4cca19070..e97593c37 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -23,11 +23,7 @@ import software.amazon.smithy.model.traits.StringTrait; import software.amazon.smithy.python.codegen.integrations.PythonIntegration; import software.amazon.smithy.python.codegen.integrations.RuntimeClientPlugin; -import software.amazon.smithy.python.codegen.sections.InitializeHttpAuthParametersSection; -import software.amazon.smithy.python.codegen.sections.ResolveEndpointSection; -import software.amazon.smithy.python.codegen.sections.ResolveIdentitySection; -import software.amazon.smithy.python.codegen.sections.SendRequestSection; -import software.amazon.smithy.python.codegen.sections.SignRequestSection; +import software.amazon.smithy.python.codegen.sections.*; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; @@ -69,10 +65,10 @@ private void generateService(PythonWriter writer) { $L :param config: Optional configuration for the client. Here you can set things like the - endpoint for HTTP services or auth credentials. + endpoint for HTTP services or auth credentials. :param plugins: A list of callables that modify the configuration dynamically. These - can be used to set defaults, for example.""", docs); + can be used to set defaults, for example.""", docs); }); var defaultPlugins = new LinkedHashSet(); @@ -797,6 +793,7 @@ private void generateOperation(PythonWriter writer, OperationShape operation) { var output = model.expectShape(operation.getOutputShape()); var outputSymbol = symbolProvider.toSymbol(output); + writer.pushState(new OperationSection(service, operation)); writer.openBlock("async def $L(self, input: $T, plugins: list[$T] | None = None) -> $T:", "", operationMethodSymbol.getName(), @@ -824,26 +821,29 @@ private void generateOperation(PythonWriter writer, OperationShape operation) { """, serSymbol, deserSymbol, operation.getId().getName()); } }); + writer.popState(); } private void writeSharedOperationInit(PythonWriter writer, OperationShape operation, Shape input) { writer.writeDocs(() -> { - var docs = operation.getTrait(DocumentationTrait.class) + var docs = writer.formatDocs(operation.getTrait(DocumentationTrait.class) .map(StringTrait::getValue) - .orElse(String.format("Invokes the %s operation.", operation.getId().getName())); + .orElse(String.format("Invokes the %s operation.", + operation.getId().getName()))); var inputDocs = input.getTrait(DocumentationTrait.class) .map(StringTrait::getValue) .orElse("The operation's input."); writer.write(""" - $L - :param input: $L :param plugins: A list of callables that modify the configuration dynamically. - Changes made by these plugins only apply for the duration of the operation - execution and will not affect any other operation invocations.""", docs, inputDocs); + Changes made by these plugins only apply for the duration of the operation + execution and will not affect any other operation invocations. + + $L + """, inputDocs, docs); }); var defaultPlugins = new LinkedHashSet(); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonDependency.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonDependency.java index 89f016b9e..908541b3f 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonDependency.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonDependency.java @@ -60,7 +60,10 @@ public enum Type { DEPENDENCY("dependency"), /** A dependency only used for testing purposes. */ - TEST_DEPENDENCY("testDependency"); + TEST_DEPENDENCY("testDependency"), + + /** A dependency only used for docs generation. */ + DOCS_DEPENDENCY("docsDependency"); private final String type; diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java index 0e1a456e1..c43e47d61 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java @@ -102,6 +102,24 @@ public final class SmithyPythonDependency { Type.TEST_DEPENDENCY, false); + /** + * library used for documentation generation + */ + public static final PythonDependency SPHINX = new PythonDependency( + "sphinx", + ">=8.2.3", + Type.DOCS_DEPENDENCY, + false); + + /** + * sphinx theme + */ + public static final PythonDependency SPHINX_PYDATA_THEME = new PythonDependency( + "pydata-sphinx-theme", + ">=0.16.1", + Type.DOCS_DEPENDENCY, + false); + private SmithyPythonDependency() {} /** diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java index 74fbffadc..cd17c6e5e 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java @@ -14,16 +14,14 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import software.amazon.smithy.aws.traits.ServiceTrait; import software.amazon.smithy.codegen.core.CodegenException; import software.amazon.smithy.codegen.core.SymbolDependency; import software.amazon.smithy.codegen.core.WriterDelegator; import software.amazon.smithy.model.traits.DocumentationTrait; import software.amazon.smithy.model.traits.StringTrait; import software.amazon.smithy.model.traits.TitleTrait; -import software.amazon.smithy.python.codegen.GenerationContext; -import software.amazon.smithy.python.codegen.PythonDependency; -import software.amazon.smithy.python.codegen.PythonSettings; -import software.amazon.smithy.python.codegen.SymbolProperties; +import software.amazon.smithy.python.codegen.*; import software.amazon.smithy.python.codegen.sections.PyprojectSection; import software.amazon.smithy.python.codegen.sections.ReadmeSection; import software.amazon.smithy.python.codegen.writer.PythonWriter; @@ -42,6 +40,7 @@ public static void generateSetup( PythonSettings settings, GenerationContext context ) { + writeDocsSkeleton(settings, context); var dependencies = gatherDependencies(context.writerDelegator().getDependencies().stream()); writePyproject(settings, context.writerDelegator(), dependencies); writeReadme(settings, context); @@ -149,9 +148,24 @@ private static void writePyproject( writer.openBlock("dependencies = [", "]", () -> writeDependencyList(writer, deps.values())); }); - Optional.ofNullable(dependencies.get(PythonDependency.Type.TEST_DEPENDENCY.getType())).ifPresent(deps -> { + Optional> testDeps = + Optional.ofNullable(dependencies.get(PythonDependency.Type.TEST_DEPENDENCY.getType())) + .map(Map::values); + + Optional> docsDeps = + Optional.ofNullable(dependencies.get(PythonDependency.Type.DOCS_DEPENDENCY.getType())) + .map(Map::values); + + if (testDeps.isPresent() || docsDeps.isPresent()) { writer.write("[project.optional-dependencies]"); - writer.openBlock("tests = [", "]", () -> writeDependencyList(writer, deps.values())); + } + + testDeps.ifPresent(deps -> { + writer.openBlock("tests = [", "]", () -> writeDependencyList(writer, deps)); + }); + + docsDeps.ifPresent(deps -> { + writer.openBlock("docs = [", "]", () -> writeDependencyList(writer, deps)); }); // TODO: remove the pyright global suppressions after the serde redo is done @@ -240,12 +254,6 @@ private static void writeReadme( """, title, description); service.getTrait(DocumentationTrait.class).map(StringTrait::getValue).ifPresent(documentation -> { - // TODO: make sure this documentation is well-formed - // Existing services in AWS, for example, have a lot of HTML docs. - // HTML nodes *are* valid commonmark technically, so it should be - // fine here. If we were to make this file RST formatted though, - // we'd have a problem. We have to solve that at some point anyway - // since the python code docs are RST format. writer.write(""" ### Documentation @@ -255,4 +263,208 @@ private static void writeReadme( writer.popState(); }); } + + /** + * Write the files required for sphinx doc generation + */ + private static void writeDocsSkeleton( + PythonSettings settings, + GenerationContext context + ) { + //TODO Add a configurable flag to disable the generation of the sphinx files + //TODO Add a configuration that will allow users to select a sphinx theme + context.writerDelegator().useFileWriter("pyproject.toml", "", writer -> { + writer.addDependency(SmithyPythonDependency.SPHINX); + writer.addDependency(SmithyPythonDependency.SPHINX_PYDATA_THEME); + }); + var service = context.model().expectShape(settings.service()); + String projectName = service.getTrait(TitleTrait.class) + .map(StringTrait::getValue) + .orElseGet(() -> service.getTrait(ServiceTrait.class) + .map(ServiceTrait::getSdkId) + .orElse(context.settings().service().getName())); + writeConf(settings, context, projectName); + writeIndexes(context, projectName); + writeMakeBat(context); + writeMakeFile(context); + } + + /** + * Write a conf.py file. + * A conf.py file is a configuration file used by Sphinx, a documentation + * generation tool for Python projects. This file contains settings and + * configurations that control the behavior and appearance of the generated + * documentation. + */ + private static void writeConf( + PythonSettings settings, + GenerationContext context, + String projectName + ) { + String version = settings.moduleVersion(); + context.writerDelegator().useFileWriter("docs/conf.py", "", writer -> { + writer.write(""" + import os + import sys + sys.path.insert(0, os.path.abspath('..')) + + project = '$L' + author = 'Amazon Web Services' + release = '$L' + + extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + ] + + templates_path = ['_templates'] + exclude_patterns = [] + + autodoc_default_options = { + 'exclude-members': 'deserialize,deserialize_kwargs,serialize,serialize_members' + } + + html_theme = 'pydata_sphinx_theme' + html_theme_options = { + "logo": { + "text": "$L", + } + } + + autodoc_typehints = 'both' + """, projectName, version, projectName); + }); + } + + /** + * Write a make.bat file. + * A make.bat file is a batch script used on Windows to build Sphinx documentation. + * This script sets up the environment and runs the Sphinx build commands. + * + * @param context The generation context containing the writer delegator. + */ + private static void writeMakeBat( + GenerationContext context + ) { + context.writerDelegator().useFileWriter("docs/make.bat", "", writer -> { + writer.write(""" + @ECHO OFF + + pushd %~dp0 + + REM Command file for Sphinx documentation + + if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build + ) + set BUILDDIR=build + set SERVICESDIR=source/reference/services + set SPHINXOPTS=-j auto + set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . + + if "%1" == "" goto help + + if "%1" == "clean" ( + rmdir /S /Q %BUILDDIR% + goto end + ) + + if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + echo. + echo "Build finished. The HTML pages are in %BUILDDIR%/html." + goto end + ) + + :help + %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + + :end + popd + """); + }); + } + + /** + * Write a Makefile. + * A Makefile is used on Unix-based systems to build Sphinx documentation. + * This file contains rules for cleaning the build directory and generating HTML documentation. + * + * @param context The generation context containing the writer delegator. + */ + private static void writeMakeFile( + GenerationContext context + ) { + context.writerDelegator().useFileWriter("docs/Makefile", "", writer -> { + writer.write(""" + SPHINXBUILD = sphinx-build + BUILDDIR = build + SERVICESDIR = source/reference/services + SPHINXOPTS = -j auto + ALLSPHINXOPTS = -d $$(BUILDDIR)/doctrees $$(SPHINXOPTS) . + + clean: + \t-rm -rf $$(BUILDDIR)/* + + html: + \t$$(SPHINXBUILD) -b html $$(ALLSPHINXOPTS) $$(BUILDDIR)/html + \t@echo + \t@echo "Build finished. The HTML pages are in $$(BUILDDIR)/html." + """); + }); + } + + /** + * Write the main index files for the documentation. + * This method creates the main index.rst file and additional index files for + * the client and models sections. + * + * @param context The generation context containing the writer delegator. + */ + private static void writeIndexes(GenerationContext context, String projectName) { + // Write the main index file for the documentation + context.writerDelegator().useFileWriter("docs/index.rst", "", writer -> { + writer.write(""" + $L + $L + + .. toctree:: + :maxdepth: 2 + :titlesonly: + :glob: + + */index + """, projectName, "=".repeat(projectName.length())); + }); + + // Write the index file for the client section + writeIndexFile(context, "docs/client/index.rst", "Client"); + + // Write the index file for the models section + writeIndexFile(context, "docs/models/index.rst", "Models"); + } + + /** + * Helper method to write an index file with the given title. + * This method creates an index file at the specified file path with the provided title. + * + * @param context The generation context. + * @param filePath The file path of the index file. + * @param title The title of the index file. + */ + private static void writeIndexFile(GenerationContext context, String filePath, String title) { + context.writerDelegator().useFileWriter(filePath, "", writer -> { + writer.write(""" + $L + ======= + .. toctree:: + :maxdepth: 1 + :titlesonly: + :glob: + + * + """, title); + }); + } + } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java index a91d63000..d7cc4dd1b 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java @@ -35,6 +35,8 @@ import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; import software.amazon.smithy.python.codegen.SymbolProperties; +import software.amazon.smithy.python.codegen.sections.ErrorSection; +import software.amazon.smithy.python.codegen.sections.StructureSection; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; @@ -98,6 +100,7 @@ public void run() { private void renderStructure() { writer.addStdlibImport("dataclasses", "dataclass"); var symbol = symbolProvider.toSymbol(shape); + writer.pushState(new StructureSection(shape)); writer.write(""" @dataclass(kw_only=True) @@ -116,6 +119,8 @@ class $L: writer.consumer(w -> writeProperties()), writer.consumer(w -> generateSerializeMethod()), writer.consumer(w -> generateDeserializeMethod())); + + writer.popState(); } private void renderError() { @@ -128,7 +133,7 @@ private void renderError() { var code = shape.getId().getName(); var symbol = symbolProvider.toSymbol(shape); var apiError = CodegenUtils.getApiError(settings); - + writer.pushState(new ErrorSection(symbol)); writer.write(""" @dataclass(kw_only=True) class $1L($2T): @@ -153,6 +158,8 @@ class $1L($2T): writer.consumer(w -> writeProperties()), writer.consumer(w -> generateSerializeMethod()), writer.consumer(w -> generateDeserializeMethod())); + writer.popState(); + } private void writeProperties() { @@ -231,7 +238,8 @@ private void writeClassDocs(boolean isError) { }); if (isError) { - writer.write(":param message: A message associated with the specific error."); + writer.write("\n:param message: A message associated with the " + + "specific error."); } if (!shape.members().isEmpty()) { diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java index e592460d9..b05c56fbb 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java @@ -15,6 +15,8 @@ import software.amazon.smithy.model.traits.StringTrait; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.SymbolProperties; +import software.amazon.smithy.python.codegen.sections.UnionMemberSection; +import software.amazon.smithy.python.codegen.sections.UnionSection; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; @@ -47,6 +49,7 @@ public UnionGenerator( @Override public void run() { + writer.addStdlibImports("typing", Set.of("Union")); writer.pushState(); var parentName = symbolProvider.toSymbol(shape).getName(); writer.addStdlibImport("dataclasses", "dataclass"); @@ -61,7 +64,7 @@ public void run() { var target = model.expectShape(member.getTarget()); var targetSymbol = symbolProvider.toSymbol(target); - + writer.pushState(new UnionMemberSection(memberSymbol)); writer.write(""" @dataclass class $1L: @@ -92,6 +95,7 @@ def deserialize(cls, deserializer: ShapeDeserializer) -> Self: new MemberDeserializerGenerator(context, w, member, "deserializer"))) ); + writer.popState(); } // Note that the unknown variant doesn't implement __eq__. This is because @@ -99,6 +103,7 @@ def deserialize(cls, deserializer: ShapeDeserializer) -> Self: // Since the underlying value is unknown and un-comparable, that is the only // realistic implementation. var unknownSymbol = symbolProvider.toSymbol(shape).expectProperty(SymbolProperties.UNION_UNKNOWN); + writer.pushState(new UnionMemberSection(unknownSymbol)); writer.addImport("smithy_core.exceptions", "SmithyException"); writer.write(""" @dataclass @@ -125,9 +130,14 @@ raise NotImplementedError() """, unknownSymbol.getName()); memberNames.add(unknownSymbol.getName()); + writer.popState(); - writer.write("type $L = $L\n", parentName, String.join(" | ", memberNames)); + writer.pushState(new UnionSection(shape, parentName, memberNames)); + // We need to use the old union syntax until we either migrate away from + // Sphinx or Sphinx fixes the issue upstream: https://github.com/sphinx-doc/sphinx/issues/10785 + writer.write("$L = Union[$L]\n", parentName, String.join(" | ", memberNames)); shape.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeDocs(trait.getValue())); + writer.popState(); generateDeserializer(); writer.popState(); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/PythonIntegration.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/PythonIntegration.java index cde5869ab..8fe18b2e7 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/PythonIntegration.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/PythonIntegration.java @@ -7,6 +7,7 @@ import java.util.Collections; import java.util.List; import software.amazon.smithy.codegen.core.SmithyIntegration; +import software.amazon.smithy.model.Model; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; import software.amazon.smithy.python.codegen.generators.ProtocolGenerator; @@ -39,6 +40,10 @@ default List getClientPlugins(GenerationContext context) { return Collections.emptyList(); } + default Model preprocessModel(Model model, PythonSettings settings) { + return model; + } + /** * Writes out all extra files required by runtime plugins. */ diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/ErrorSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/ErrorSection.java new file mode 100644 index 000000000..4b8831f0f --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/ErrorSection.java @@ -0,0 +1,15 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * A section that controls writing an error. + */ +@SmithyInternalApi +public record ErrorSection(Symbol errorSymbol) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/OperationSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/OperationSection.java new file mode 100644 index 000000000..ceab19545 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/OperationSection.java @@ -0,0 +1,16 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * A section that controls writing an operation. + */ +@SmithyInternalApi +public record OperationSection(Shape service, OperationShape operation) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/StructureSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/StructureSection.java new file mode 100644 index 000000000..23713e651 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/StructureSection.java @@ -0,0 +1,15 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * A section that controls writing a structure. + */ +@SmithyInternalApi +public record StructureSection(StructureShape structure) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionMemberSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionMemberSection.java new file mode 100644 index 000000000..bd6d999e6 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionMemberSection.java @@ -0,0 +1,15 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.sections; + +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * A section that controls writing a union member. + */ +@SmithyInternalApi +public record UnionMemberSection(Symbol memberSymbol) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionSection.java new file mode 100644 index 000000000..55138f2f6 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionSection.java @@ -0,0 +1,19 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.sections; + +import java.util.ArrayList; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.utils.CodeSection; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * A section that controls writing a union. + */ +@SmithyInternalApi +public record UnionSection( + UnionShape unionShape, + String parentName, + ArrayList memberNames) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java new file mode 100644 index 000000000..811d938c0 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java @@ -0,0 +1,175 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.writer; + +import static org.jsoup.nodes.Document.OutputSettings.Syntax.html; + +import org.commonmark.node.BlockQuote; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.Heading; +import org.commonmark.node.HtmlBlock; +import org.commonmark.node.ListBlock; +import org.commonmark.node.ThematicBreak; +import org.commonmark.parser.Parser; +import org.commonmark.renderer.html.HtmlRenderer; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.nodes.Node; +import org.jsoup.nodes.TextNode; +import org.jsoup.select.NodeVisitor; +import software.amazon.smithy.utils.SetUtils; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Add a runtime plugin to convert the HTML docs that are provided by services into RST + */ +@SmithyInternalApi +public class MarkdownToRstDocConverter { + private static final Parser MARKDOWN_PARSER = Parser.builder() + .enabledBlockTypes(SetUtils.of( + Heading.class, + HtmlBlock.class, + ThematicBreak.class, + FencedCodeBlock.class, + BlockQuote.class, + ListBlock.class)) + .build(); + + // Singleton instance + private static final MarkdownToRstDocConverter DOC_CONVERTER = new MarkdownToRstDocConverter(); + + // Private constructor to prevent instantiation + private MarkdownToRstDocConverter() { + // Constructor + } + + public static MarkdownToRstDocConverter getInstance() { + return DOC_CONVERTER; + } + + public String convertCommonmarkToRst(String commonmark) { + String html = + HtmlRenderer.builder().escapeHtml(false).build().render(MARKDOWN_PARSER.parse(commonmark)); + Document document = Jsoup.parse(html); + RstNodeVisitor visitor = new RstNodeVisitor(); + document.body().traverse(visitor); + return "\n" + visitor; + } + + private static class RstNodeVisitor implements NodeVisitor { + //TODO migrate away from StringBuilder to use a SimpleCodeWriter + private final StringBuilder sb = new StringBuilder(); + private boolean inList = false; + private int listDepth = 0; + + @Override + public void head(Node node, int depth) { + if (node instanceof TextNode) { + TextNode textNode = (TextNode) node; + String text = textNode.text(); + if (!text.trim().isEmpty()) { + sb.append(text); + // Account for services making a paragraph tag that's empty except + // for a newline + } else if (node.parent() instanceof Element && ((Element) node.parent()).tagName().equals("p")) { + sb.append(text.replaceAll("[ \\t]+", "")); + } + } else if (node instanceof Element) { + Element element = (Element) node; + switch (element.tagName()) { + case "a": + sb.append("`"); + break; + case "b": + case "strong": + sb.append("**"); + break; + case "i": + case "em": + sb.append("*"); + break; + case "code": + sb.append("``"); + break; + case "important": + sb.append("\n.. important::\n "); + break; + case "note": + sb.append("\n.. note::\n "); + break; + case "ul": + inList = true; + listDepth++; + sb.append("\n"); + break; + case "li": + if (inList) { + sb.append(" ".repeat(listDepth - 1)).append("* "); + } + break; + case "h1": + sb.append("\n"); + break; + default: + break; + } + } + } + + @Override + public void tail(Node node, int depth) { + if (node instanceof Element) { + Element element = (Element) node; + switch (element.tagName()) { + case "a": + sb.append(" <").append(element.attr("href")).append(">`_"); + break; + case "b": + case "strong": + sb.append("**"); + break; + case "i": + case "em": + sb.append("*"); + break; + case "code": + sb.append("``"); + break; + case "important": + case "note": + case "p": + sb.append("\n"); + break; + case "ul": + listDepth--; + if (listDepth == 0) { + inList = false; + } + if (sb.charAt(sb.length() - 1) != '\n') { + sb.append("\n\n"); + } + break; + case "li": + if (sb.charAt(sb.length() - 1) != '\n') { + sb.append("\n\n"); + } + break; + case "h1": + String title = element.text(); + sb.append("\n").append("=".repeat(title.length())).append("\n"); + break; + default: + break; + } + } + } + + @Override + public String toString() { + return sb.toString(); + } + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java index 20c00f885..0e5901139 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java @@ -39,9 +39,10 @@ public final class PythonWriter extends SymbolWriter { private static final Logger LOGGER = Logger.getLogger(PythonWriter.class.getName()); + private static final MarkdownToRstDocConverter DOC_CONVERTER = MarkdownToRstDocConverter.getInstance(); private final String fullPackageName; - private final boolean addCodegenWarningHeader; + private final String commentStart; private boolean addLogger = false; /** @@ -51,7 +52,7 @@ public final class PythonWriter extends SymbolWriter MAX_LINE_LENGTH) { + int wrapAt = findWrapPosition(line, MAX_LINE_LENGTH); + wrappedText.append(indentStr).append(line, 0, wrapAt).append("\n"); + line = line.substring(wrapAt).trim(); + if (line.isEmpty()) { + return; + } + } + wrappedText.append(indentStr).append(line).append("\n"); + } + + private static int findWrapPosition(String line, int maxLineLength) { + // Find the last space before maxLineLength + int wrapAt = line.lastIndexOf(' ', maxLineLength); + if (wrapAt == -1) { + // If no space found, don't wrap + wrapAt = line.length(); + } else { + // Ensure we don't break a link + int linkStart = line.lastIndexOf("`", wrapAt); + int linkEnd = line.indexOf("`_", wrapAt); + if (linkStart != -1 && (linkEnd != -1 && linkEnd > linkStart)) { + linkEnd = line.indexOf("`_", linkStart); + if (linkEnd != -1) { + wrapAt = linkEnd + 2; + } else { + // No matching `_` found, keep the original wrap position + wrapAt = line.lastIndexOf(' ', maxLineLength); + if (wrapAt == -1) { + wrapAt = maxLineLength; + } + } + } + } + // Include trailing punctuation before a space in the previous line + int nextSpace = line.indexOf(' ', wrapAt); + if (nextSpace != -1) { + int i = wrapAt; + while (i < nextSpace && !Character.isLetterOrDigit(line.charAt(i))) { + i++; + } + if (i == nextSpace) { + wrapAt = nextSpace; + } + } + return wrapAt; + } + + private static int getIndentationLevel(String line) { + int indent = 0; + while (indent < line.length() && Character.isWhitespace(line.charAt(indent))) { + indent++; + } + return indent; } /** @@ -288,8 +369,8 @@ public String toString() { } contents += super.toString(); - if (addCodegenWarningHeader) { - String header = "# Code generated by smithy-python-codegen DO NOT EDIT.\n\n"; + if (!commentStart.equals("")) { + String header = String.format("%s Code generated by smithy-python-codegen DO NOT EDIT.%n%n", commentStart); contents = header + contents; } return contents; diff --git a/codegen/gradle/libs.versions.toml b/codegen/gradle/libs.versions.toml index b4513c2a1..94f940712 100644 --- a/codegen/gradle/libs.versions.toml +++ b/codegen/gradle/libs.versions.toml @@ -6,6 +6,8 @@ spotbugs = "6.0.22" spotless = "7.0.2" smithy-gradle-plugins = "1.2.0" dep-analysis = "2.12.0" +jsoup = "1.19.1" +commonmark = "0.15.2" [libraries] smithy-model = { module = "software.amazon.smithy:smithy-model", version.ref = "smithy" } @@ -28,6 +30,9 @@ spotbugs = { module = "com.github.spotbugs.snom:spotbugs-gradle-plugin", version spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.ref = "spotless" } dependency-analysis = { module = "com.autonomousapps:dependency-analysis-gradle-plugin", version.ref = "dep-analysis" } +jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } +commonmark = { module = "com.atlassian.commonmark:commonmark", version.ref ="commonmark" } + [plugins] smithy-gradle-base = { id = "software.amazon.smithy.gradle.smithy-base", version.ref = "smithy-gradle-plugins" } smithy-gradle-jar = { id = "software.amazon.smithy.gradle.smithy-jar", version.ref = "smithy-gradle-plugins" }