doc: Generate develop/api/overview.rst API table from doxygen

A new extension, api_overview.py, is used to, leveraging doxygen's
Python module doxmlparser, parse the doxygen generated XML files.
All groups ('defgroup' and 'addtogroup' tags) are collected, alongside
their 'version' and 'since' info.

From there, a new Sphinx directive `api-overview-table` is populated,
including the name of the group, and if available, their 'since' and
'version' information.

Signed-off-by: Ederson de Souza <ederson.desouza@intel.com>
Signed-off-by: Anas Nashif <anas.nashif@intel.com>

doc/_extensions/zephyr/api_overview.py

@@ -0,0 +1,172 @@
+# Copyright (c) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import doxmlparser
+
+from docutils import nodes
+from doxmlparser.compound import DoxCompoundKind
+from pathlib import Path
+from sphinx.application import Sphinx
+from sphinx.util.docutils import SphinxDirective
+from typing import Any, Dict
+
+
+class ApiOverview(SphinxDirective):
+    """
+    This is a Zephyr directive to generate a table containing an overview
+    of all APIs. This table will show the API name, version and since which
+    version it is present - all information extracted from Doxygen XML output.
+
+    It is exclusively used by the doc/develop/api/overview.rst page.
+
+    Configuration options:
+
+    api_overview_doxygen_xml_dir: Doxygen xml output directory
+    api_overview_doxygen_base_url: Doxygen base html directory
+    """
+
+    def run(self):
+        return [self.env.api_overview_table]
+
+
+def get_group(innergroup, all_groups):
+    try:
+        return [
+            g
+            for g in all_groups
+            if g.get_compounddef()[0].get_id() == innergroup.get_refid()
+        ][0]
+    except IndexError as e:
+        raise Exception(f"Unexpected group {innergroup.get_refid()}") from e
+
+
+def visit_group(app, group, all_groups, rows, indent=0):
+    version = since = ""
+    cdef = group.get_compounddef()[0]
+
+    ssects = [
+        s for p in cdef.get_detaileddescription().get_para() for s in p.get_simplesect()
+    ]
+    for sect in ssects:
+        if sect.get_kind() == "since":
+            since = sect.get_para()[0].get_valueOf_()
+        elif sect.get_kind() == "version":
+            version = sect.get_para()[0].get_valueOf_()
+
+    if since:
+        since_url = nodes.inline()
+        reference = nodes.reference(text=f"v{since.strip()}.0", refuri=f"https://github.com/zephyrproject-rtos/zephyr/releases/tag/v{since.strip()}.0")
+        reference.attributes["internal"] = True
+        since_url += reference
+    else:
+        since_url = nodes.Text("")
+
+    url_base = Path(app.config.api_overview_doxygen_base_url)
+    url = url_base / f"{cdef.get_id()}.html"
+
+    title = cdef.get_title()
+
+    row_node = nodes.row()
+
+    # Next entry will contain the spacer and the link with API name
+    entry = nodes.entry()
+    span = nodes.Text("".join(["_"] * indent) + " ")
+    entry += span
+
+    # API name with link
+    inline = nodes.inline()
+    reference = nodes.reference(text=title, refuri=str(url))
+    reference.attributes["internal"] = True
+    inline += reference
+    entry += inline
+    row_node += entry
+
+    version_node = nodes.Text(version)
+    # Finally, add version and since
+    for cell in [version_node, since_url]:
+        entry = nodes.entry()
+        entry += cell
+        row_node += entry
+    rows.append(row_node)
+
+    for innergroup in cdef.get_innergroup():
+        visit_group(
+            app, get_group(innergroup, all_groups), all_groups, rows, indent + 2
+        )
+
+
+def parse_xml_dir(dir_name):
+    groups = []
+    root = doxmlparser.index.parse(Path(dir_name) / "index.xml", True)
+    for compound in root.get_compound():
+        if compound.get_kind() == DoxCompoundKind.GROUP:
+            file_name = Path(dir_name) / f"{compound.get_refid()}.xml"
+            groups.append(doxmlparser.compound.parse(file_name, True))
+
+    return groups
+
+
+def generate_table(app, toplevel, groups):
+    table = nodes.table()
+    tgroup = nodes.tgroup()
+
+    thead = nodes.thead()
+    thead_row = nodes.row()
+    for header_name in ["API", "Version", "Available in Zephyr Since"]:
+        colspec = nodes.colspec()
+        tgroup += colspec
+
+        entry = nodes.entry()
+        entry += nodes.Text(header_name)
+        thead_row += entry
+    thead += thead_row
+    tgroup += thead
+
+    rows = []
+    tbody = nodes.tbody()
+    for t in toplevel:
+        visit_group(app, t, groups, rows)
+    tbody.extend(rows)
+    tgroup += tbody
+
+    table += tgroup
+
+    return table
+
+
+def sync_contents(app: Sphinx) -> None:
+    if app.config.doxyrunner_outdir:
+        doxygen_out_dir = Path(app.config.doxyrunner_outdir)
+    else:
+        doxygen_out_dir = Path(app.outdir) / "_doxygen"
+
+    doxygen_xml_dir = doxygen_out_dir / "xml"
+    groups = parse_xml_dir(doxygen_xml_dir)
+
+    toplevel = [
+        g
+        for g in groups
+        if g.get_compounddef()[0].get_id()
+        not in [
+            i.get_refid()
+            for h in [j.get_compounddef()[0].get_innergroup() for j in groups]
+            for i in h
+        ]
+    ]
+
+    app.builder.env.api_overview_table = generate_table(app, toplevel, groups)
+
+
+def setup(app) -> Dict[str, Any]:
+    app.add_config_value("api_overview_doxygen_xml_dir", "html/doxygen/xml", "env")
+    app.add_config_value("api_overview_doxygen_base_url", "../../doxygen/html", "env")
+
+    app.add_directive("api-overview-table", ApiOverview)
+
+    app.connect("builder-inited", sync_contents)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

doc/conf.py

@@ -95,6 +95,7 @@
     "sphinx_togglebutton",
     "zephyr.external_content",
     "zephyr.domain",
+    "zephyr.api_overview",
 ]
 
 # Only use SVG converter when it is really needed, e.g. LaTeX.
@@ -364,6 +365,9 @@
 linkcheck_workers = 10
 linkcheck_anchors = False
 
+# -- Options for zephyr.api_overview --------------------------------------
+
+api_overview_doxygen_base_url = "../../doxygen/html"
 
 def setup(app):
     # theme customizations

doc/develop/api/api_lifecycle.rst

@@ -20,6 +20,9 @@ no longer optimal or supported by the underlying platforms.
 An up-to-date table of all APIs and their maturity level can be found in the
 :ref:`api_overview` page.
 
+
+.. _api_lifecycle_experimental:
+
 Experimental
 *************
 
@@ -36,6 +39,10 @@ The following requirements apply to all new APIs:
   of said API (in the case of peripheral APIs, this corresponds to one driver)
 - At least one sample using the new API (may only build on one single board)
 
+When introducing a new and experimental API, mark the API version in the headers
+where the API is defined. An experimental API shall have a version where the minor
+version is up to one (0.1.z). (see `api overview <api_overview>`)
+
 Peripheral APIs (Hardware Related)
 ==================================
 
@@ -46,13 +53,19 @@ the Architecture working group consisting of representatives from different vend
 The API shall be promoted to ``unstable`` when it has at least two
 implementations on different hardware platforms.
 
+.. _api_lifecycle_unstable:
+
 Unstable
 ********
 
 The API is in the process of settling, but has not yet had sufficient real-world
 testing to be considered stable. The API is considered generic in nature and can
 be used on different hardware platforms.
 
+When the API changes status to unstable API, mark the API version in the headers
+where the API is defined. Unstable APIs shall have a version where the minor
+version is larger than one (0.y.z | y > 1 ). (see `api overview <api_overview>`)
+
 .. note::
 
    Changes will not be announced.
@@ -69,6 +82,8 @@ Hardware Agnostic APIs
 For hardware agnostic APIs, multiple applications using it are required to
 promote an API from ``experimental`` to ``unstable``.
 
+.. _api_lifecycle_stable:
+
 Stable
 *******
 
@@ -94,6 +109,11 @@ In order to declare an API ``stable``, the following steps need to be followed:
    `Zephyr Architecture meeting`_ where, barring any objections, the Pull Request
    will be merged
 
+
+When the API changes status to stable API, mark the API version in the headers
+where the API is defined. Stable APIs shall have a version where the major
+version is one or larger (x.y.z | x >= 1 ). (see `api overview <api_overview>`)
+
 .. _breaking_api_changes:
 
 Introducing breaking API changes
@@ -175,6 +195,11 @@ for it to be discussed and ultimately even voted on in the `Zephyr TSC meeting`_
 If the Pull Request is merged then an email must be sent to the ``devel`` and
 ``user`` mailing lists informing them of the change.
 
+The API version shall be changed to signal backward incompatible changes. This
+is achieved by incrementing the major version (X.y.z | X > 1).  It MAY also
+include minor and patch level changes. Patch and minor versions MUST be reset to
+0 when major version is incremented. (see `api overview <api_overview>`)
+
 .. note::
 
    Breaking API changes will be listed and described in the migration guide.