Use simpler structure for the generated API reference (#949)

Use the same generation script as CTranslate2.

CHANGELOG.md

@@ -5,7 +5,7 @@ OpenNMT-tf follows [semantic versioning 2.0.0](https://semver.org/). The API cov
 * command line options
 * configuration files
 * checkpoints of non experimental models
-* classes and functions documented on the [online documentation](https://opennmt.net/OpenNMT-tf/package/opennmt.html) and directly accessible from the top-level `opennmt` package
+* classes and functions documented on the [online documentation](https://opennmt.net/OpenNMT-tf/package/overview.html) and directly accessible from the top-level `opennmt` package
 
 ---
 
@@ -537,7 +537,7 @@ See the [2.0 Transition Guide](docs/v2_transition.md) for details about the foll
 * The `onmt-main` script now makes use of subparsers which require to move the run type and it specific options to the end of the command
 * Some predefined models have been renamed or changed, see the [transition guide](docs/v2_transition.md#changed-predefined-models)
 * Some parameters in the YAML configuration have been renamed or changed, see the [transition guide](docs/v2_transition.md#changed-parameters)
-* A lot of public classes and functions have changed, see the [API documentation](http://opennmt.net/OpenNMT-tf/package/opennmt.html) for details
+* A lot of public classes and functions have changed, see the [API documentation](http://opennmt.net/OpenNMT-tf/package/overview.html) for details
 * TFRecord files generated with the `opennmt.inputters.write_sequence_record` function or the `onmt-ark-to-records` script are no longer compatible and should be re-generated
 
 This version also changes the public API scope of the project:
@@ -548,7 +548,7 @@ This version also changes the public API scope of the project:
 ### New features
 
 * Object-based layers extending `tf.keras.layers.Layer`
-* Many new reusable modules and layers, see the [API documentation](http://opennmt.net/OpenNMT-tf/package/opennmt.html)
+* Many new reusable modules and layers, see the [API documentation](http://opennmt.net/OpenNMT-tf/package/overview.html)
 * Replace `tf.estimator` by custom loops for more control and clearer execution path
 * Multi-GPU training with `tf.distribute`
 * Support early stopping based on any evaluation metrics

README.md

@@ -37,7 +37,7 @@ opennmt.models.SequenceToSequence(
 )
 ```
 
-The [`opennmt`](https://opennmt.net/OpenNMT-tf/package/opennmt.html) package exposes other building blocks that can be used to design:
+The [`opennmt`](https://opennmt.net/OpenNMT-tf/package/overview.html) package exposes other building blocks that can be used to design:
 
 * [multiple input features](https://opennmt.net/OpenNMT-tf/package/opennmt.inputters.ParallelInputter.html)
 * [mixed embedding representation](https://opennmt.net/OpenNMT-tf/package/opennmt.inputters.MixedInputter.html)
@@ -128,7 +128,7 @@ onmt-main --model_type <model> --config <config_file.yml> --auto_config <run_typ
 
 ### Library
 
-OpenNMT-tf also exposes [well-defined and stable APIs](https://opennmt.net/OpenNMT-tf/package/opennmt.html), from high-level training utilities to low-level model layers and dataset transformations.
+OpenNMT-tf also exposes [well-defined and stable APIs](https://opennmt.net/OpenNMT-tf/package/overview.html), from high-level training utilities to low-level model layers and dataset transformations.
 
 For example, the `Runner` class can be used to train and evaluate models with few lines of code:
 
@@ -177,7 +177,7 @@ More examples using OpenNMT-tf as a library can be found online:
 * The directory [examples/library](https://github.com/OpenNMT/OpenNMT-tf/tree/master/examples/library) contains additional examples that use OpenNMT-tf as a library
 * [nmt-wizard-docker](https://github.com/OpenNMT/nmt-wizard-docker) uses the high-level `opennmt.Runner` API to wrap OpenNMT-tf with a custom interface for training, translating, and serving
 
-*For a complete overview of the APIs, see the [package documentation](https://opennmt.net/OpenNMT-tf/package/opennmt.html).*
+*For a complete overview of the APIs, see the [package documentation](https://opennmt.net/OpenNMT-tf/package/overview.html).*
 
 ## Additional resources
 

docs/conf.py

@@ -15,8 +15,7 @@
 version = ".".join(release.split(".")[:2])  # The short X.Y version.
 
 extensions = [
-    "recommonmark",
-    "sphinx_markdown_tables",
+    "myst_parser",
     "sphinx.ext.autodoc",
     "sphinx.ext.napoleon",
     "sphinx.ext.todo",

docs/generate-apidoc.py

@@ -8,8 +8,9 @@
 
 def document_class(output_dir, class_path, base_path=None, children_paths=None):
     with open(os.path.join(output_dir, "%s.rst" % class_path), "w") as doc:
-        doc.write("%s\n" % class_path)
-        doc.write("=" * len(class_path))
+        class_name = class_path.split(".")[-1]
+        doc.write("%s\n" % class_name)
+        doc.write("=" * len(class_name))
         doc.write("\n\n")
         doc.write(".. autoclass:: %s\n" % class_path)
         doc.write("    :members:\n")
@@ -24,8 +25,9 @@ def document_class(output_dir, class_path, base_path=None, children_paths=None):
 
 def document_function(output_dir, function_path):
     with open(os.path.join(output_dir, "%s.rst" % function_path), "w") as doc:
-        doc.write("%s\n" % function_path)
-        doc.write("=" * len(function_path))
+        function_name = function_path.split(".")[-1]
+        doc.write("%s\n" % function_name)
+        doc.write("=" * len(function_name))
         doc.write("\n\n")
         doc.write(".. autofunction:: %s\n" % function_path)
 
@@ -80,7 +82,8 @@ def annotate_classes(classes):
 
 def document_module(module, module_path, module_map, output_dir):
     if not module_is_public(module):
-        return False
+        return []
+    modules = [module_path]
     submodules = []
     classes = []
     functions = []
@@ -92,37 +95,26 @@ def document_module(module, module_path, module_map, output_dir):
         symbol_path = "%s.%s" % (module_path, symbol_name)
         if inspect.isclass(symbol):
             classes.append((symbol, symbol_path))
-        elif inspect.isfunction(symbol) or inspect.ismethod(symbol):
+        elif (
+            inspect.isfunction(symbol)
+            or inspect.ismethod(symbol)
+            or inspect.isroutine(symbol)
+        ):
             functions.append(symbol_path)
         elif inspect.ismodule(symbol):
             submodules.append((symbol_path, symbol))
         elif isinstance(symbol, (numbers.Number, str)):
             constants.append(symbol_path)
 
     with open(os.path.join(output_dir, "%s.rst" % module_path), "w") as doc:
-        doc.write("%s module\n" % module_path)
-        doc.write("=" * (len(module_path) + 7))
+        doc.write("%s\n" % module_path)
+        doc.write("=" * (len(module_path)))
         doc.write("\n\n")
         doc.write(".. automodule:: %s\n\n" % module_path)
 
-        if submodules:
-            submodules = list(
-                filter(
-                    lambda x: document_module(x[1], x[0], module_map, output_dir),
-                    submodules,
-                )
-            )
-            if submodules:
-                doc.write("Submodules\n")
-                doc.write("----------\n\n")
-                doc.write(".. toctree::\n\n")
-                for module_path, module in submodules:
-                    doc.write("   %s\n" % module_path)
+        doc.write(".. toctree::\n\n")
 
         if classes:
-            doc.write("\nClasses\n")
-            doc.write("-------\n\n")
-            doc.write(".. toctree::\n\n")
             for class_info in annotate_classes(classes):
                 base = class_info["parent"]
                 base_path = module_map.get(
@@ -146,23 +138,31 @@ def document_module(module, module_path, module_map, output_dir):
                 )
 
         if functions:
-            doc.write("\nFunctions\n")
-            doc.write("---------\n\n")
-            doc.write(".. toctree::\n\n")
             for function_path in functions:
                 doc.write("   %s\n" % function_path)
                 document_function(output_dir, function_path)
 
         if constants:
-            doc.write("\nConstants\n")
-            doc.write("---------\n\n")
             for constant_path in constants:
                 doc.write("* %s\n" % constant_path)
 
-        return True
+        if submodules:
+            for submodule_path, submodule in submodules:
+                modules.extend(
+                    document_module(submodule, submodule_path, module_map, output_dir)
+                )
+
+        return modules
 
 
 output_dir = sys.argv[1]
 os.makedirs(output_dir)
 module_map = get_module_map(opennmt, "opennmt")
-document_module(opennmt, "opennmt", module_map, output_dir)
+module_paths = document_module(opennmt, "opennmt", module_map, output_dir)
+
+with open(os.path.join(output_dir, "overview.rst"), "w") as overview:
+    overview.write("Python\n")
+    overview.write("======\n\n")
+    overview.write(".. toctree::\n\n")
+    for module_path in module_paths:
+        overview.write("   %s\n" % module_path)

docs/index.rst

@@ -38,5 +38,4 @@ Index
    :caption: API
    :maxdepth: 1
 
-   api_examples.md
-   package/opennmt
+   package/overview

setup.py

@@ -11,8 +11,7 @@
     "pytest-cov",
 ]
 docs_require = [
-    "recommonmark==0.7.*",
-    "sphinx-markdown-tables==0.0.15",
+    "myst-parser==0.17.*",
     "sphinx-rtd-theme==1.0.*",
     "sphinx==4.5.*",
 ]