Merge pull request #108 from khaeru/enh/write-report

Improve write_report, compat using @singledispatch

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.6.1
+  rev: v1.7.0
   hooks:
   - id: mypy
     additional_dependencies:

doc/api.rst

@@ -287,6 +287,7 @@ Top-level classes and functions
 
 .. autoclass:: genno.Quantity
    :members:
+   :inherited-members: pipe, shape, size
 
 The :class:`.Quantity` constructor converts its arguments to an internal, :class:`xarray.DataArray`-like data format:
 
@@ -307,7 +308,7 @@ Common :mod:`genno` usage, e.g. in :mod:`message_ix`, creates large, sparse data
 The goal is that all :mod:`genno`-based code, including built-in and user functions, can treat quantity arguments as if they were :class:`~xarray.DataArray`.
 
 .. automodule:: genno
-   :members: MissingKeyError
+   :members: ComputationError, KeyExistsError, MissingKeyError
 
 Operators
 =========
@@ -424,3 +425,6 @@ Utilities for testing
 .. automodule:: genno.testing
    :members:
    :exclude-members: parametrize_quantity_class
+
+.. automodule:: genno.testing.jupyter
+   :members:

doc/cache.rst

@@ -31,7 +31,7 @@ A cache key is computed from:
 
 1. the name of `func`.
 2. the arguments to `func`, and
-3. the compiled bytecode of `func` (see :func:`hash_code`).
+3. the compiled bytecode of `func` (see :func:`.hash_code`).
 
 If a file exists in ``cache_path`` with a matching key, it is loaded and returned instead of calling `func`.
 
@@ -61,20 +61,20 @@ Consider a function that loads a very large file, or performs some slow processi
         # … further processing …
         return Quantity(result)
 
-We want to cache the result of :func:`slow_data_load`, but have the cache refreshed when the file contents change.
+We want to cache the result of :py:`slow_data_load`, but have the cache refreshed when the file contents change.
 We do this using the `_extra_cache_key` argument to the function.
 This argument is not used in the function, but *does* affect the value of the cache key.
 
 When calling the function, pass some value that indicates whether the contents of `path` have changed.
-One possibility is the modification time, via :meth:`.Path.stat`:
+One possibility is the modification time, via :meth:`pathlib.Path.stat`:
 
 .. code-block:: python
 
     def load_cached_1(path):
         return slow_data_load(path, path.stat().st_mtime)
 
 Another possibility is to hash the entire file.
-:func:`hash_contents` is provided for this purpose:
+:func:`.hash_contents` is provided for this purpose:
 
 .. code-block:: python
 

doc/compat-pyam.rst

@@ -45,10 +45,8 @@ Pyam (:mod:`.compat.pyam`)
 
       as_pyam
       add_as_pyam
-      concat
-      write_report
 
-   .. .. autofunction:: as_pyam
+   This module also registers implementations of :func:`.concat` and :func:`.write_report` that handle :class:`pyam.IamDataFrame` objects.
 
    .. autofunction:: add_as_pyam
 

doc/conf.py

@@ -26,22 +26,21 @@
     "sphinx.ext.viewcode",
 ]
 
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
-
 # List of patterns, relative to source directory, that match files and directories to
 # ignore when looking for source files. This pattern also affects html_static_path and
 # html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
+nitpicky = True
 
 rst_prolog = """
 .. role:: py(code)
    :language: python
-
-.. |KeyLike| replace:: :obj:`~genno.core.key.KeyLike`
 """
 
+# Paths that contain templates, relative to the current directory.
+templates_path = ["_templates"]
+
 
 def setup(app):
     """Modify the sphinx.ext.autodoc config to handle Operators as functions."""
@@ -89,19 +88,41 @@ def can_document_member(cls, member, membername, isattr, parent) -> bool:
 # -- Options for sphinx.ext.intersphinx ------------------------------------------------
 
 intersphinx_mapping = {
-    "dask": ("https://docs.dask.org/en/stable/", None),
+    "dask": ("https://docs.dask.org/en/stable", None),
     "ixmp": ("https://docs.messageix.org/projects/ixmp/en/latest", None),
-    "joblib": ("https://joblib.readthedocs.io/en/latest/", None),
-    "graphviz": ("https://graphviz.readthedocs.io/en/stable/", None),
+    "joblib": ("https://joblib.readthedocs.io/en/latest", None),
+    "graphviz": ("https://graphviz.readthedocs.io/en/stable", None),
     "message_ix": ("https://docs.messageix.org/en/latest", None),
-    "pandas": ("https://pandas.pydata.org/docs/", None),
-    "pint": ("https://pint.readthedocs.io/en/stable/", None),
-    "plotnine": ("https://plotnine.readthedocs.io/en/stable/", None),
-    "pyam": ("https://pyam-iamc.readthedocs.io/en/stable/", None),
-    "python": ("https://docs.python.org/3/", None),
+    "message-ix-models": ("https://docs.messageix.org/projects/models/en/latest", None),
+    "nbclient": ("https://nbclient.readthedocs.io/en/latest", None),
+    "nbformat": ("https://nbformat.readthedocs.io/en/latest", None),
+    "numpy": ("https://numpy.org/doc/stable", None),
+    "pandas": ("https://pandas.pydata.org/docs", None),
+    "pint": ("https://pint.readthedocs.io/en/stable", None),
+    "plotnine": ("https://plotnine.readthedocs.io/en/stable", None),
+    "pyam": ("https://pyam-iamc.readthedocs.io/en/stable", None),
+    "python": ("https://docs.python.org/3", None),
     "sdmx1": ("https://sdmx1.readthedocs.io/en/stable", None),
     "sparse": ("https://sparse.pydata.org/en/stable", None),
-    "xarray": ("https://docs.xarray.dev/en/stable/", None),
+    "xarray": ("https://docs.xarray.dev/en/stable", None),
+}
+
+# -- Options for sphinx.ext.napoleon ---------------------------------------------------
+
+napoleon_preprocess_types = True
+napoleon_type_aliases = {
+    # Standard library
+    "callable": "typing.Callable",
+    "collection": "collections.abc.Collection",
+    "hashable": "collections.abc.Hashable",
+    "iterable": "collections.abc.Iterable",
+    "mapping": "collections.abc.Mapping",
+    "sequence": "collections.abc.Sequence",
+    "Path": "pathlib.Path",
+    # This package
+    "KeyLike": "genno.core.key.KeyLike",
+    # Others
+    "Code": "sdmx.model.common.Code",
 }
 
 # -- Options for sphinx.ext.todo -------------------------------------------------------

doc/index.rst

@@ -4,7 +4,7 @@
 **genno** is a Python package for describing and executing complex calculations on labelled, multi-dimensional data.
 It aims to make these calculations efficient, transparent, and easily validated as part of scientific research.
 
-genno is built on high-quality Python data packages including ``dask``, ``xarray``, and ``pint``; and provides (current or planned) compatibility with packages including ``pandas``, ``matplotlib``, ``plotnine``, ``ixmp``, ``sdmx1``, and ``pyam``.
+genno is built on high-quality Python data packages including :py:`dask`, :mod:`xarray`, and ```pint``; and provides (current or planned) compatibility with packages including :mod:`pandas`, :mod:`matplotlib`, :mod:`plotnine <.compat.plotnine>`, :mod:`ixmp`, :mod:`sdmx1 <.compat.sdmx>`, and :mod:`pyam <.compat.pyam>`.
 
 .. toctree::
    :maxdepth: 2
@@ -22,9 +22,9 @@ Compatibility
 
 :mod:`.genno` provides built-in support for interaction with:
 
-- :doc:`Plotnine <compat-plotnine>` (:mod:`.plotnine`), via :mod:`.compat.plotnine`.
-- :doc:`Pyam <compat-pyam>` (:mod:`.pyam`), via :mod:`.compat.pyam`.
-- :doc:`SDMX <compat-sdmx>` (:mod:`.sdmx`), via :mod:`.compat.sdmx`.
+- :doc:`Plotnine <compat-plotnine>` (:mod:`plotnine`), via :mod:`.compat.plotnine`.
+- :doc:`Pyam <compat-pyam>` (:mod:`pyam`), via :mod:`.compat.pyam`.
+- :doc:`SDMX <compat-sdmx>` (:mod:`sdmx`), via :mod:`.compat.sdmx`.
 
 .. toctree::
    :maxdepth: 1
@@ -37,8 +37,8 @@ Compatibility
 
 Packages that extend :mod:`genno` include:
 
-- :mod:`ixmp.reporting`
-- :mod:`message_ix.reporting`
+- :mod:`ixmp.report`
+- :mod:`message_ix.report`
 
 .. toctree::
    :maxdepth: 2

doc/whatsnew.rst

@@ -1,8 +1,12 @@
 What's new
 **********
 
-.. Next release
-.. ============
+Next release
+============
+
+- :func:`.write_report` and :func:`.concat` are single-dispatch functions for simpler extension in user code (:pull:`108`).
+- New argument to :func:`.write_report`: :py:`kwargs`, including "header_comment" to write a header comment at the start of a :file:`.csv` file (:pull:`108`).
+- Fix many cross-references in the documentation (:pull:`108`).
 
 v1.20.0 (2023-10-28)
 ====================
@@ -123,7 +127,7 @@ v1.17.0 (2023-05-15)
 ====================
 
 - Bug fix: genno v1.16.1 (:pull:`85`) introduced :class:`ValueError` for some usages of :func:`.computations.sum <.operator.sum>` (:pull:`88`).
-- Provide typed signatures for :meth:`Quantity.bfill`, :meth:`~Quantity.cumprod`, :meth:`~Quantity.ffill` for the benefit of downstream applications (:pull:`88`).
+- Provide typed signatures for :meth:`.Quantity.bfill`, :meth:`~.Quantity.cumprod`, :meth:`~.Quantity.ffill` for the benefit of downstream applications (:pull:`88`).
 - Ensure and test that :attr:`.Quantity.name` and :attr:`~.Quantity.units` pass through all :mod:`~genno.computations <genno.operator>`, in particular :func:`~.operator.aggregate`, :func:`~.operator.convert_units`, and :func:`~.operator.sum` (:pull:`88`).
 - Simplify arithmetic operations (:func:`~.operator.div`, :func:`~.operator.mul`, :func:`~.operator.pow`) so they are agnostic as to the :class:`.Quantity` class in use (:pull:`88`).
 - Ensure :attr:`.AttrSeries.index` is always :class:`pandas.MultiIndex` (:pull:`88`).
@@ -144,13 +148,13 @@ v1.16.0 (2023-04-29)
 - genno supports and is tested on Python 3.11 (:pull:`83`).
 - Update dependencies (:pull:`83`):
 
-  - General: :mod:`importlib_resources` (the independent backport of :mod:`importlib.resources`) is added for Python 3.9 and earlier.
+  - General: :py:`importlib_resources` (the independent backport of :mod:`importlib.resources`) is added for Python 3.9 and earlier.
   - ``genno[sparse]``: new set of optional dependencies, including :mod:`sparse`.
     Install this set in order to use :class:`.SparseDataArray` for :class:`.Quantity`.
 
-    Note that sparse depends on :mod:`numba`, and thus :mod:`llvmlite`, and both of these package can lag new Python versions by several months.
+    Note that sparse depends on :py:`numba`, and thus :py:`llvmlite`, and both of these package can lag new Python versions by several months.
     For example, as of this release, they do not yet support Python 3.11, and thus :mod:`sparse` and :class:`.SparseDataArray` can only be used with Python 3.10 and earlier.
-  - ``genno[tests]``: :mod:`ixmp` is removed; :mod:`jupyter` and :mod:`nbclient` are added.
+  - ``genno[tests]``: :mod:`ixmp` is removed; :py:`jupyter` and :py:`nbclient` are added.
     Testing utilities in :mod:`genno.testing.jupyter` are duplicated from :mod:`ixmp.testing.jupyter`.
 
 - Adjust :meth:`.AttrSeries.interp` for compatibility with pandas 2.0.0 (released 2023-04-03) (:pull:`81`).
@@ -268,7 +272,7 @@ v1.9.1 (2022-01-27)
 Note that installing ``genno[pyam]`` (including via ``genno[compat]``) currently forces the installation of an old version of :mod:`pint`; version 0.17 or earlier.
 Users wishing to use :mod:`genno.compat.pyam` should first install ``genno[pyam]``, then ``pip install --upgrade pint`` to restore a recent version of pint (0.18 or newer) that is usable with genno.
 
-- :func:`computations.concat` works with :class:`.AttrSeries` with misaligned dimensions (:pull:`53`).
+- :func:`.computations.concat <.operator.concat>` works with :class:`.AttrSeries` with misaligned dimensions (:pull:`53`).
 - Improve typing of :class:`.Quantity` and :class:`.Computer` to help with using `mypy <https://mypy.readthedocs.io>`_ on code that uses :mod:`genno` (:pull:`53`).
 
 v1.9.0 (2021-11-23)
@@ -307,7 +311,7 @@ v1.7.0 (2021-07-22)
 v1.6.0 (2021-07-07)
 ===================
 
-- Add :meth:`Key.permute_dims` (:pull:`47`).
+- Add :py:`Key.permute_dims()` (:pull:`47`).
 - Improve performance of :meth:`.Computer.check_keys` (:pull:`47`).
 
 v1.5.2 (2021-07-06)
@@ -323,9 +327,9 @@ v1.5.1 (2021-07-01)
 v1.5.0 (2021-06-27)
 ===================
 
-- Adjust :meth:`.test_assign_coords` for xarray 0.18.2 (:pull:`43`).
+- Adjust :func:`.test_assign_coords` for xarray 0.18.2 (:pull:`43`).
 - Make :attr:`.Key.dims` order-insensitive so that ``Key("foo", "ab") == Key("foo", "ba")`` (:pull:`42`); make corresponding changes to :class:`.Computer` (:pull:`44`).
-- Fix “:class:`AttributeError`: 'COO' object has no attribute 'item'” on :meth:`SparseDataArray.item` (:pull:`41`).
+- Fix “:class:`AttributeError`: 'COO' object has no attribute 'item'” on :meth:`.SparseDataArray.item` (:pull:`41`).
 
 v1.4.0 (2021-04-26)
 ===================
@@ -339,18 +343,18 @@ v1.3.0 (2021-03-22)
 
 - Bump minimum version of :mod:`sparse` from 0.10 to 0.12 and adjust to changes in this version (:pull:`39`)
 
-  - Remove :meth:`.SparseDataArray.equals`, obviated by improvements in :mod:`sparse`.
+  - Remove :py:`SparseDataArray.equals()`, obviated by improvements in :mod:`sparse`.
 
 - Improve :class:`.AttrSeries` (:pull:`39`)
 
   - Implement :meth:`~.AttrSeries.drop_vars` and :meth:`~.AttrSeries.expand_dims`.
   - :meth:`~.AttrSeries.assign_coords` can relabel an entire dimension.
-  - :meth:`~.AttrSeries.sel` can accept :class:`.DataArray` indexers and rename/combine dimensions.
+  - :meth:`~.AttrSeries.sel` can accept :class:`~xarray.DataArray` indexers and rename/combine dimensions.
 
 v1.2.1 (2021-03-08)
 ===================
 
-- Bug fix: Provide abstract :class:`.Quantity.to_series` method for type checking in packages that depend on :mod:`genno`.
+- Bug fix: Provide abstract :meth:`.Quantity.to_series` method for type checking in packages that depend on :mod:`genno`.
 
 v1.2.0 (2021-03-08)
 ===================
@@ -377,15 +381,15 @@ v1.1.0 (2021-02-16)
 v1.0.0 (2021-02-13)
 ===================
 
-- Adjust for usage by :mod:`ixmp.reporting` and :mod:`message_ix.reporting` (:pull:`28`):
+- Adjust for usage by :mod:`ixmp.reporting <ixmp.report>` and :mod:`message_ix.reporting <message_ix.report>` (:pull:`28`):
 
   - Reduce minimum Python version to 3.6.
     This is lower than the minimum version for xarray (3.7), but matches ixmp, etc.
-  - Remove :mod:`compat.ixmp`; this code has been moved to :mod:`ixmp.reporting`, replacing what was there.
-    Likewise, remove :mod:`compat.message_ix`.
+  - Remove submodule :py:`compat.ixmp`; this code has been moved to :mod:`ixmp.reporting <ixmp.report>`, replacing what was there.
+    Likewise, remove submodule :py:`compat.message_ix`.
   - Simplify the form & parsing of ``iamc:`` section entries in configuration files:
 
-    - Remove unused feature to add :func:`group_sum` to the chain of tasks.
+    - Remove unused feature to add :py:`group_sum()` to the chain of tasks.
     - Keys now conform more closely to the arguments of :meth:`.Computer.convert_pyam`.
 
   - Move argument-checking from :func:`.as_pyam` to :meth:`.convert_pyam()`.
@@ -416,11 +420,11 @@ v0.2.0 (2021-01-18)
 -------------------
 
 - Increase test coverage to 100% (:pull:`12`).
-- Port code from :mod:`message_ix.reporting` (:pull:`11`).
+- Port code from :mod:`message_ix.reporting <message_ix.report>` (:pull:`11`).
 - Add :mod:`.compat.pyam`.
 - Add a `name` parameter to :func:`.load_file`.
 
 v0.1.0 (2021-01-10)
 -------------------
 
-- Initial code port from :mod:`ixmp.reporting`.
+- Initial code port from :mod:`ixmp.reporting <ixmp.report>`.

genno/caching.py

@@ -53,7 +53,7 @@ def ignore(cls, *types):
 
     @classmethod
     def register(cls, func):
-        """Register a `func` to serialize a type not handled by :class:`.JSONEncoder`.
+        """Register `func` to serialize a type not handled by :class:`json.JSONEncoder`.
 
         `func` should return a type that *is* handled by JSONEncoder; see the docs.
 
@@ -69,12 +69,12 @@ def register(cls, func):
         return _encode.register(func)
 
     def default(self, o):
-        """For `o`, return an object serializable by the base :class:`.JSONEncoder`.
+        """For `o`, return an object serializable by the base :class:`json.JSONEncoder`.
 
         - :class:`pathlib.Path`: the string representation of `o`.
-        - :class:`code` objects (from Python's built-in :mod:`inspect` module), e.g.
-          a function or lambda: :class:`blake2b` hash of the object's bytecode and its
-          serialized constants.
+        - :ref:`python:code-objects` (from Python's built-in :mod:`inspect` module), for
+          instance a function or lambda: :func:`~hashlib.blake2b` hash of the object's
+          bytecode and its serialized constants.
 
           .. warning:: This is not 100% guaranteed to change if the operation of `o` (or
              other code called in turn by `o`) changes. If relying on this behaviour,
@@ -102,7 +102,7 @@ def default(self, o):
 
 
 def hash_args(*args, **kwargs):
-    """Return a 20-character :class:`blake2b` hex digest of `args` and `kwargs`.
+    """Return a 20-character :func:`hashlib.blake2b` hex digest of `args` and `kwargs`.
 
     Used by :func:`.decorate`.
 
@@ -121,7 +121,7 @@ def hash_args(*args, **kwargs):
 
 
 def hash_code(func: Callable) -> str:
-    """Return the :class:`.blake2b` hex digest of the compiled bytecode of `func`.
+    """Return the :func:`hashlib.blake2b` hex digest of the compiled bytecode of `func`.
 
     See also
     --------
@@ -133,11 +133,11 @@ def hash_code(func: Callable) -> str:
 
 
 def hash_contents(path: Union[Path, str], chunk_size=65536) -> str:
-    """Return the :class:`.blake2b` hex digest of the contents of the file at `path`.
+    """Return the :func:`hashlib.blake2b` hex digest the file contents of `path`.
 
     Parameters
     ----------
-    chunk_size : int, *optional*
+    chunk_size : int, optional
         Read the file in chunks of this size; default 64 kB.
     """
     with Path(path).open("rb") as f:
@@ -154,13 +154,13 @@ def decorate(
 
     Parameters
     ----------
-    computer : Computer, *optional*
+    computer : .Computer, optional
         If supplied, the ``config`` dictionary stored in the Computer is used to look
         up values for `cache_path` and `cache_skip`, at the moment when `func` is
         called.
-    cache_path : os.Pathlike, *optional*
+    cache_path : os.PathLike, optional
         Directory in which to store cache files.
-    cache_skip : bool, *optional*
+    cache_skip : bool, optional
         If :obj:`True`, ignore existing cache entries and overwrite them with new
         values from `func`.