coverage: Specify modules to run coverage for

Currently, the coverage builder lets you check for partially documented
modules, but there is no mechanism to identify totally undocumented
modules. Resolve this by introducing a new 'coverage_modules' config
option. This is a list of modules that should be documented somewhere
within the documentation tree. Any modules that are specified in the
configuration value but are not documented anywhere will result in a
warning. Likewise, any modules that are not in the config option but are
documented somewhere will result in a warning.

Signed-off-by: Stephen Finucane <stephen@that.guru>

doc/usage/extensions/coverage.rst

@@ -6,15 +6,58 @@
 
 This extension features one additional builder, the :class:`CoverageBuilder`.
 
+.. todo:: Write this section.
+
+.. note::
+
+   The :doc:`sphinx-apidoc </man/sphinx-apidoc>` command can be used to
+   automatically generate API documentation for all code in a project, avoiding
+   the need to manually author these documents and keep them up-to-date.
+
+.. warning::
+
+   :mod:`~sphinx.ext.coverage` **imports** the modules to be documented.  If any
+   modules have side effects on import, these will be executed by the coverage
+   builder when ``sphinx-build`` is run.
+
+   If you document scripts (as opposed to library modules), make sure their main
+   routine is protected by a ``if __name__ == '__main__'`` condition.
+
+.. note::
+
+   For Sphinx (actually, the Python interpreter that executes Sphinx) to find
+   your module, it must be importable.  That means that the module or the
+   package must be in one of the directories on :data:`sys.path` -- adapt your
+   :data:`sys.path` in the configuration file accordingly.
+
+To use this builder, activate the coverage extension in your configuration file
+and give ``-b coverage`` on the command line.
+
+
+Builder
+-------
+
 .. class:: CoverageBuilder
 
-   To use this builder, activate the coverage extension in your configuration
-   file and give ``-b coverage`` on the command line.
 
-.. todo:: Write this section.
+Configuration
+-------------
+
+Several configuration values can be used to specify what the builder should
+check:
+
+.. confval:: coverage_modules
+
+   List of Python packages or modules to test coverage for. When this is
+   provided, Sphinx will introspect each package or module provided in this
+   list as well as all sub-packages and sub-modules found in each. When this is
+   not provided, Sphinx will only provide coverage for Python packages and
+   modules that it is aware of: that is, any modules documented using the
+   :rst:dir:`py:module` directive provided in the :ref:`Python domain
+   <python-domain>` or the :rst:dir:`automodule` directive provided by the
+   :mod:`~sphinx.ext.autodoc` extension.
 
-Several configuration values can be used to specify what the builder
-should check:
+   .. versionadded: 7.2
 
 .. confval:: coverage_ignore_modules
 

sphinx/ext/coverage.py

@@ -9,6 +9,7 @@
 import glob
 import inspect
 import pickle
+import pkgutil
 import re
 import sys
 from importlib import import_module
@@ -65,6 +66,45 @@ def _add_row(col_widths: list[int], columns: list[str], separator: str) -> Itera
     yield _add_line(col_widths, separator)
 
 
+def load_modules(mod_name: str, ignored_module_exps: list[re.Pattern[str]]) -> set[str]:
+    """Recursively load all submodules.
+
+    :param mod_name: The name of a module to load submodules for.
+    :param ignored_module_exps: A list of regexes for modules to ignore.
+    :returns: A set of modules names including the provided module name,
+        ``mod_name``
+    :raises ImportError: If the module indicated by ``mod_name`` could not be
+        loaded.
+    """
+    modules: set[str] = set()
+
+    for exp in ignored_module_exps:
+        if exp.match(mod_name):
+            return modules
+
+    # this can raise an exception but it's the responsibility of the caller to
+    # handle this
+    mod = import_module(mod_name)
+
+    modules.add(mod_name)
+
+    for sub_mod_info in pkgutil.iter_modules(mod.__path__):
+        if sub_mod_info.name == '__main__':
+            continue
+
+        # TODO: Do we want to skip private modules (note: different from private
+        # objects)?
+        # if sub_mod_info.name.startswith('_'):
+        #     continue
+
+        if sub_mod_info.ispkg:
+            modules |= load_modules(f'{mod_name}.{sub_mod_info.name}', ignored_module_exps)
+        else:
+            modules.add(f'{mod_name}.{sub_mod_info.name}')
+
+    return modules
+
+
 class CoverageBuilder(Builder):
     """
     Evaluates coverage of code in the documentation.
@@ -90,6 +130,7 @@ def init(self) -> None:
         for (name, exps) in self.config.coverage_ignore_c_items.items():
             self.c_ignorexps[name] = compile_regex_list('coverage_ignore_c_items',
                                                         exps)
+        self.modules = self.config.coverage_modules
         self.mod_ignorexps = compile_regex_list('coverage_ignore_modules',
                                                 self.config.coverage_ignore_modules)
         self.cls_ignorexps = compile_regex_list('coverage_ignore_classes',
@@ -167,11 +208,43 @@ def ignore_pyobj(self, full_name: str) -> bool:
         )
 
     def build_py_coverage(self) -> None:
-        objects = self.env.domaindata['py']['objects']
-        modules = self.env.domaindata['py']['modules']
+        seen_objects = self.env.domaindata['py']['objects']
+        seen_modules = self.env.domaindata['py']['modules']
 
         skip_undoc = self.config.coverage_skip_undoc_in_source
 
+        # Figure out which of the two operating modes to use:
+        #
+        # - If 'coverage_modules' is not specified, we check coverage for all modules
+        #   seen in the documentation tree. Any objects found in these modules that are
+        #   not documented will be noted. This will therefore only identify missing
+        #   objects but it requires no additional configuration.
+        # - If 'coverage_modules' is specified, we check coverage for all modules
+        #   specified in this configuration value. Any objects found in these modules
+        #   that are not documented will be noted. In addition, any objects from other
+        #   modules that are documented will be noted. This will therefore identify both
+        #   missing modules and missing objects but it requires manual configuration.
+        if not self.modules:
+            modules = set(seen_modules)
+        else:
+            modules = set()
+            for mod_name in self.modules:
+                try:
+                    modules |= load_modules(mod_name, self.mod_ignorexps)
+                except ImportError as err:
+                    logger.warning(__('module %s could not be imported: %s'), mod_name, err)
+                    self.py_undoc[mod_name] = {'error': err}
+                    continue
+
+            # if there are additional modules then we warn (but still scan)
+            additional_modules = set(seen_modules) - modules
+            if additional_modules:
+                logger.warning(
+                    __('the following modules are documented but were not specified '
+                       'in coverage_modules: %s'),
+                    ', '.join(additional_modules),
+                )
+
         for mod_name in modules:
             ignore = False
             for exp in self.mod_ignorexps:
@@ -211,7 +284,7 @@ def build_py_coverage(self) -> None:
                     continue
 
                 if inspect.isfunction(obj):
-                    if full_name not in objects:
+                    if full_name not in seen_objects:
                         for exp in self.fun_ignorexps:
                             if exp.match(name):
                                 break
@@ -227,7 +300,7 @@ def build_py_coverage(self) -> None:
                         if exp.match(name):
                             break
                     else:
-                        if full_name not in objects:
+                        if full_name not in seen_objects:
                             if skip_undoc and not obj.__doc__:
                                 continue
                             # not documented at all
@@ -255,7 +328,7 @@ def build_py_coverage(self) -> None:
                             full_attr_name = f'{full_name}.{attr_name}'
                             if self.ignore_pyobj(full_attr_name):
                                 continue
-                            if full_attr_name not in objects:
+                            if full_attr_name not in seen_objects:
                                 attrs.append(attr_name)
                                 undocumented_objects.add(full_attr_name)
                             else:
@@ -385,6 +458,7 @@ def finish(self) -> None:
 
 def setup(app: Sphinx) -> dict[str, Any]:
     app.add_builder(CoverageBuilder)
+    app.add_config_value('coverage_modules', [], False)
     app.add_config_value('coverage_ignore_modules', [], False)
     app.add_config_value('coverage_ignore_functions', [], False)
     app.add_config_value('coverage_ignore_classes', [], False)