Merge pull request #60 from khaeru/feature/graph-class

Index keys for improved performance

.github/workflows/lint.yaml

@@ -18,14 +18,13 @@ jobs:
     - uses: actions/setup-python@v2
       # Uncomment and set this to match the "Latest version testable on GitHub
       # Actions" in pytest.yaml, if that is not the default used by setup-python
-      with:
-       python-version: "3.9"
+      # with:
+      #   python-version: "3.9"
 
     - name: Cache Python packages
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
-        path: |
-          ~/.cache/pip
+        path: ~/.cache/pip
         key: lint-${{ runner.os }}
 
     - name: Upgrade pip, wheel, setuptools-scm

.github/workflows/publish.yaml

@@ -21,10 +21,9 @@ jobs:
       #   python-version: "3.9"
 
     - name: Cache Python packages
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
-        path: |
-          ~/.cache/pip
+        path: ~/.cache/pip
         key: publish-${{ runner.os }}
 
     - name: Upgrade pip, wheel, setuptools-scm
@@ -36,15 +35,15 @@ jobs:
         twine check dist/*
 
     - name: Publish to TestPyPI
-      uses: pypa/gh-action-pypi-publish@v1.4.1
+      uses: pypa/gh-action-pypi-publish@v1.5.0
       if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
       with:
         user: __token__
         password: ${{ secrets.TESTPYPI_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.4.1
+      uses: pypa/gh-action-pypi-publish@v1.5.0
       if: github.event_name == 'release'
       with:
         user: __token__

.github/workflows/pytest.yaml

@@ -23,24 +23,29 @@ jobs:
         python-version:
         - "3.7"  # Earliest version supported by genno; matches xarray/setup.cfg
         - "3.8"
-        - "3.9"  # Latest testable on GitHub Actions
+        - "3.9"
+        - "3.10" # Latest release / latest supported by genno / testable on GHA
 
-        # For new releases or development versions of Python, compiled binary
-        # wheels may not be available for some dependencies, e.g. llvmlite,
+        # For fresh releases and development versions of Python, compiled
+        # binary wheels are not available for some dependencies, e.g. llvmlite,
         # numba, numpy, and/or pandas. Compiling these on the job runner
         # requires a more elaborate build environment, currently out of scope
         # for genno. Exclude these versions from CI.
-        # - "3.10"  # Latest Python release / latest supported by genno
         # - "3.11.0-alpha.2"  # Development version
 
+        exclude:
+        # JPype1 binary wheels are not available for this combination
+        - os: windows-latest
+          python-version: "3.10"
+
       fail-fast: false
 
     runs-on: ${{ matrix.os }}
     name: ${{ matrix.os }}-py${{ matrix.python-version }}
 
     steps:
     - name: Cancel previous runs that have not completed
-      uses: styfle/cancel-workflow-action@0.7.0
+      uses: styfle/cancel-workflow-action@0.9.1
       with:
         access_token: ${{ github.token }}
 
@@ -55,13 +60,14 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Locate pip cache directory
+      id: pip-cache
+      run: echo "::set-output name=dir::$(pip cache dir)"
+
     - name: Cache Python packages
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
-        path: |
-          ~/.cache/pip
-          ~/Library/Caches/pip
-          ~/appdata/local/pip/cache
+        path: ${{ steps.pip-cache.outputs.dir }}
         key: ${{ matrix.os }}-py${{ matrix.python-version }}
         restore-keys: |
           ${{ matrix.os }}-
@@ -72,10 +78,14 @@ jobs:
       run: python -m pip install --upgrade pip wheel setuptools-scm
 
     - name: Install Python package and dependencies
-      # pyam-iamc (IAMconsortium/pyam#589) forces pint 0.17; override
       run: |
         pip install --editable .[docs,tests]
-        pip install --upgrade pint
+
+    - name: Specify version of pint
+      if: matrix.python-version != '3.7'
+      run: |
+        # Branch for with https://github.com/hgrecco/pint/pull/1508
+        pip install --upgrade git+https://github.com/khaeru/pint.git@issue/1498#egg=pint
 
     - name: Run test suite using pytest
       run: pytest genno --trace-config --verbose --cov-report=xml --cov-report=term --color=yes
@@ -86,4 +96,4 @@ jobs:
       run: make html
 
     - name: Upload test coverage to Codecov.io
-      uses: codecov/codecov-action@v1.2.1
+      uses: codecov/codecov-action@v2

doc/api.rst

@@ -289,21 +289,21 @@ Computations
 
    Genno's :ref:`compatibility modules <compat>` each provide additional computations.
 
-   Calculations:
+   Numerical calculations:
 
    .. autosummary::
       add
       aggregate
-      apply_units
       broadcast_map
       combine
       disaggregate_shares
+      div
       group_sum
       interpolate
+      mul
       pow
       product
       ratio
-      select
       sum
 
    Input and output:
@@ -315,7 +315,11 @@ Computations
    Data manipulation:
 
    .. autosummary::
+      apply_units
       concat
+      relabel
+      rename_dims
+      select
 
 
 Internal format for quantities
@@ -340,6 +344,9 @@ Internal format for quantities
 Utilities
 =========
 
+.. automodule:: genno.core.graph
+   :members:
+
 .. automodule:: genno.util
    :members:
 

doc/conf.py

@@ -54,6 +54,7 @@
     "pull": ("https://github.com/khaeru/genno/pull/%s", "PR #"),
     "gh-user": ("https://github.com/%s", "@"),
 }
+extlinks_detect_hardcoded_links = False
 
 # -- Options for sphinx.ext.intersphinx ------------------------------------------------
 

doc/config.rst

@@ -132,6 +132,8 @@ The :func:`.handles` decorator can be used to mark a custom function that handle
 Specific sections
 =================
 
+.. _config-aggregate:
+
 ``aggregate:``
 --------------
 
@@ -149,6 +151,8 @@ Each entry contains:
    New tag to append to the keys for the aggregated quantities.
 ``_dim:`` (:class:`str`)
    Dimensions on which to aggregate.
+``_fail:`` (:class:`str` or :class:`int`)
+   Action or log level when aggregation fails; corresponds to the `fail` argument to :meth:`.Computer.aggregate`.
 
 Note the leading underscores.
 This is to distinguish these from all other keys, which are treated as group names.
@@ -285,6 +289,8 @@ If the ``path:`` key is a relative path, it is resolved relative to the director
       dims: {i: i, j_dim: j}
 
 
+.. _config-general:
+
 ``general:``
 ------------
 
@@ -301,8 +307,10 @@ Each item contains:
   E.g. if "product", then :meth:`.Computer.add_product` is called, which also automatically infers the correct dimensions for each input.
 ``key:``
    The key for the computed quantity.
+
+   If the key has the single dimension "\*" (e.g. ``key: "X:*:tag"``), then dimensions of the resulting key are inferred to be the union of the dimensions of each of the ``inputs:``, below.
 ``inputs:``
-   A list of keys to which the computation is applied.
+   A list of (1 or more) keys to which the computation is applied.
 ``args:`` (:class:`dict`, optional)
    Keyword arguments to the computation.
 ``add args:`` (:class:`dict`, optional)

doc/whatsnew.rst

@@ -6,8 +6,42 @@ What's new
    :backlinks: none
    :depth: 1
 
-.. Next release
-.. ============
+Next release
+============
+
+Migration notes
+---------------
+
+The `index` keyword argument to :meth:`.Computer.add_single` / :meth:`.add` is deprecated (:pull:`60`) and will be removed in or after v3.0.0.
+Indexing behaviour changes slightly: :meth:`.Computer.full_key` always returns the :class:`.Key` which matches its arguments and has the *greatest number of dimensions*.
+For instance:
+
+.. code-block:: python
+
+    c.add_product("foo", "bar:a-b-c", "baz:x-y-z", sums=True)
+
+…will generate a key ``<foo:a-b-c-x-y-z>`` and all partial sums over subsets of its dimensions; ``c.full_key("foo")`` will return this key.
+
+Care should be taken to avoid adding 2+ keys with the same name, tag, **and** number of dimensions:
+
+.. code-block:: python
+
+   c.add("foo:a-b-c", ...)
+   c.add("foo:l-m-n", ...)
+   c.add("foo:x-y-z", ...)
+
+This situation is ambiguous and the behaviour of :meth:`Computer.full_key` is undefined.
+Instead, add a :attr:`~.Key.tag` to disambiguate.
+
+All changes
+-----------
+
+- :meth:`.Key.product` accepts :class:`str` arguments (:pull:`60`).
+- New class method :meth:`.Key.bare_name` (:pull:`60`).
+- Infer dimensions for e.g. ``X:*:tag`` in :ref:`config-general` configuration items (:pull:`60`).
+- Handle the `fail` argument to :meth:`.Computer.aggregate` through :ref:`config-aggregate` configuration items (:pull:`60`).
+- New computations :func:`.relabel` and :func:`.rename_dims` (:pull:`60`).
+- Improve perfomance for adding large number of computations to :class:`.Computer` (:pull:`60`).
 
 v1.10.0 (2022-03-31)
 ====================

genno/compat/pyam/computations.py

@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 from typing import Callable, Collection, Optional, Union
 
 import pyam
@@ -90,20 +91,22 @@ def concat(*args, **kwargs):
         return genno.computations.concat(*args, **kwargs)
 
 
-def write_report(quantity, path):
-    """Write the report identified by *key* to the file at *path*.
+def write_report(obj, path: Union[str, Path]) -> None:
+    """Write  obj` to the file at `path`.
 
-    If *quantity* is a :class:`pyam.IamDataFrame` and *path* ends with '.csv' or
-    '.xlsx', use :mod:`pyam` methods to write the file to CSV or Excel format,
-    respectively. Otherwise, equivalent to :func:`genno.computations.write_report`.
+    If `obj` is a :class:`pyam.IamDataFrame` and `path` ends with ".csv" or ".xlsx",
+    use :mod:`pyam` methods to write the file to CSV or Excel format, respectively.
+    Otherwise, equivalent to :func:`genno.computations.write_report`.
     """
-    if not isinstance(quantity, pyam.IamDataFrame):
-        return genno.computations.write_report(quantity, path)
+    if not isinstance(obj, pyam.IamDataFrame):
+        return genno.computations.write_report(obj, path)
+
+    path = Path(path)
 
     if path.suffix == ".csv":
-        quantity.to_csv(path)
+        obj.to_csv(path)
     elif path.suffix == ".xlsx":
-        quantity.to_excel(path, merge_cells=False)
+        obj.to_excel(path, merge_cells=False)
     else:
         raise ValueError(
             f"pyam.IamDataFrame can be written to .csv or .xlsx, not {path.suffix}"