Merge pull request #156 from iiasa/enh/2024-W10

Miscellaneous improvements for 2024-W10
iiasa · Mar 26, 2024 · c7938e8 · c7938e8
2 parents 9be2032 + 3b7bb24
commit c7938e8
Show file tree

Hide file tree

Showing 62 changed files with 981 additions and 502 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,27 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    # Only update major versions
+    ignore:
+    - dependency-name: "*"
+      update-types:
+      - "version-update:semver-minor"
+      - "version-update:semver-patch"
+    # Below config mirrors the example at
+    # https://github.com/dependabot/dependabot-core/blob/main/.github/dependabot.yml
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "16:00"
+    groups:
+      all-actions:
+        patterns: [ "*" ]
+    assignees:
+    - "glatterf42"
+    - "khaeru"
+    labels:
+    - "dependencies"
+    reviewers:
+    - "glatterf42"
+    - "khaeru"
diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
@@ -16,31 +16,55 @@ jobs:
   pytest:
     strategy:
       matrix:
-        # One job per OS.
-        version:
-        - { os: macos-latest }
-        - { os: ubuntu-latest }
-        - { os: windows-latest }
-        # Versions of both ixmp and message_ix to use plus latest supported python version
+        os:
+        - macos-latest
+        - ubuntu-latest
+        - windows-latest
         upstream:
-        - { version: v3.4.0, extra-deps: '"pandas<2.0"', python-version: "3.11" }  # Minimum version given in pyproject.toml
-        - { version: v3.5.0, extra-deps: '"pandas<2.0"', python-version: "3.11" }
-        - { version: v3.6.0, extra-deps: '"pandas<2.0"', python-version: "3.11" }
-        - { version: v3.7.0, extra-deps: "", python-version: "3.11" }  
-        - { version: v3.8.0, extra-deps: "", python-version: "3.12" } # Latest released version
-        - { version: main, extra-deps: "", python-version: "3.12" }  # Development version
+        # In each group:
+        # - Versions of ixmp and message_ix to test.
+        # - Latest supported Python version for those or other dependencies.
+        # - Extra dependencies, in particular fixed/maximum versions to resolve conficts.
+        #   - dask[dataframe] >= 2024.3.0 requires dask-expr and in turn pandas >= 2.0.
+        #     https://github.com/iiasa/message-ix-models/pull/156#issuecomment-2020152360
+        #   - genno: upstream versions < 3.8.0 import genno.computations, removed in 1.25.0.
+        #     https://github.com/iiasa/message-ix-models/pull/156
+        #   - pytest: upstream versions < 3.9.0 use a hook argument removed in pytest 8.1.0.
+        #     https://github.com/iiasa/message-ix-models/pull/155
+        #
+        # Minimum version given in pyproject.toml
+        - version: v3.4.0
+          python-version: "3.11"
+          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
+        - version: v3.5.0
+          python-version: "3.11"
+          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
+        - version: v3.6.0
+          python-version: "3.11"
+          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
+        - version: v3.7.0
+          python-version: "3.11"
+          extra-deps: 'dask[dataframe]   "genno < 1.25"                "pytest == 8.0.0"' #
+        # Latest released version
+        - version: v3.8.0
+          python-version: "3.12"
+          extra-deps: 'dask[dataframe]                                 "pytest == 8.0.0"' #
+        # Development version
+        - version: main
+          python-version: "3.12"
+          extra-deps: 'dask[dataframe]' #
 
       fail-fast: false
 
-    runs-on: ${{ matrix.version.os }}
-    name: ${{ matrix.version.os }}-py${{ matrix.upstream.python-version }}-upstream-${{ matrix.upstream.version }}
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }}-py${{ matrix.upstream.python-version }}-upstream-${{ matrix.upstream.version }}
 
     steps:
     - name: Cache test data
       uses: actions/cache@v4
       with:
         path: message-local-data
-        key: ${{ matrix.version.os }}
+        key: ${{ matrix.os }}
 
     - name: Check out message-ix-models
       uses: actions/checkout@v4
@@ -63,11 +87,6 @@ jobs:
       with:
         macos-skip-brew-update: true
 
-    - name: Install specific version of pytest
-      # All versions prior to v3.9.0 use a hook argument that pytest removed in v8.1.0
-      if: ${{ matrix.version != 'main' }}
-      run: pip install pytest==8.0.0
-
     - name: Install packages and dependencies
       # By default, install:
       # - ixmp, message_ix: from GitHub branches/tags per matrix.upstream-version (above)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -10,7 +10,7 @@ repos:
     language: python
     entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@; python -m pip list"
     additional_dependencies:
-    - mypy >= 1.8.0
+    - mypy >= 1.9.0
     - plotnine
     - pytest
     - sdmx1
@@ -20,7 +20,7 @@ repos:
     - "message-ix @ git+https://github.com/iiasa/message_ix.git@main"
     args: ["."]
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.13
+  rev: v0.3.2
   hooks:
   - id: ruff
   - id: ruff-format

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -3,4 +3,8 @@ prune message_ix_models/data/test/iea
 prune message_ix_models/data/test/MESSAGEix-GLOBIOM_1.1_R11_no-policy_baseline
 prune message_ix_models/data/test/ssp
 prune message_ix_models/data/water/*
+# Larger package data
+# - Not distributed on PyPI.
+# - Should be fetched with Pooch from GitHub.
+exclude message_ix_models/data/ssp/*.gz
 exclude message_ix_models/data/water/*.tar.xz
diff --git a/doc/api/util.rst b/doc/api/util.rst
@@ -34,6 +34,7 @@ Commonly used:
    check_support
    convert_units
    copy_column
+   datetime_now_with_tz
    ffill
    identify_nodes
    iter_keys
@@ -52,6 +53,7 @@ Commonly used:
    same_node
    same_time
    series_of_pint_quantity
+   show_versions
 
 .. automodule:: message_ix_models.util
    :members:

diff --git a/doc/cli.rst b/doc/cli.rst
@@ -8,16 +8,16 @@ Our goal is that the *semantics* of all commands are similar, so that interactin
 .. contents::
    :local:
 
-
 Controlling CLI behaviour
 =========================
 
 To support a variety of complex use-cases, the MESSAGEix stack takes configuration and inputs from several places:
 
 :mod:`ixmp` configuration file: :file:`config.json`
 ---------------------------------------------------
+
 :mod:`ixmp` keeps track of named Platforms and their associated databases, and stores information in its :file:`config.json` file.
-See :mod:`ixmp.config`.
+See :ref:`ixmp:configuration` in the documentation.
 List existing platforms::
 
     $ ixmp platform list
@@ -27,15 +27,29 @@ To add a specific database, you can use the ixmp CLI [1]_::
     $ ixmp platform add [PLATFORMNAME] jdbc oracle [COMPUTER]:[PORT]/[SERVICENAME] [USERNAME] [PASSWORD]
 
 You may also want to make this the *default* platform.
-Unless told otherwise, :mod:`message_ix_models` creates :class:`~ixmp.Platform` objects without any arguments (``mp = ixmp.Platform()``); this loads the default platform.
+Unless told otherwise, :mod:`message_ix_models` creates :class:`~ixmp.Platform` objects without any arguments (:py:`mp = ixmp.Platform()`); this loads the default platform.
 Set the default::
 
     $ ixmp platform add default [PLATFORMNAME]
 
-:mod:`message_ix` stores only one configuration value in :file:`config.json`: ``'message model dir'``, the path to the GAMS model files.
-MESSAGEix-GLOBIOM uses the GAMS model files from the current :mod:`message_ix` ``master`` branch, so you should not set this, or unset it when using :mod:`message_ix_models`.
+:mod:`message_ix` recognizes the following :file:`config.json` value:
+
+``message_model_dir``
+   Path to the GAMS model files.
+   Most code in MESSAGEix-GLOBIOM expects the GAMS model files from the current :mod:`message_ix` ``main`` branch, so you should not set this, or unset it when using :mod:`message_ix_models`.
+
+:mod:`message_ix_models` recognizes the following 2 :file:`config.json` values:
+
+``message_local_data``
+   Path to local data, if it is set and not overridden.
+``no_message_data``
+   If not set or :any:`False`, then the CLI displays a warning message if the private :mod:`message_data` package is not installed::
 
-:mod:`message_ix_models` will use the :file:`config.json` value ``"message_local_data"`` for local data, if it is set and not overridden.
+      Warning: message_data is not installed or cannot be imported; see the documentation via --help
+
+   If set to :any:`True`, then the message is suppressed::
+
+      $ mix-models config set no_message_data true
 
 .. [1] ``[COMPUTER]`` is in this case either the hostname or the IP address.
 
@@ -46,7 +60,6 @@ For example, ixmp responds to ``IXMP_DATA``, which tells it where to find the fi
 
 :mod:`message_ix_models` responds to ``MESSAGE_LOCAL_DATA``; see :ref:`the discussion of local data <local-data>`.
 
-
 CLI parameters (arguments and options)
 --------------------------------------
 
@@ -63,12 +76,12 @@ Consider the following examples::
 
 In these examples:
 
-- ``--opt0`` is an option that (potentially) affects **any** command, including the subcommands ``cmd2`` or ``cmd3``.
-- ``--opt1`` and ``arg1`` are an option and mandatory argument to the command ``cmd1``.
-  They might not have any relevance to other ``mix-data`` commands.
-- ``cmd2`` and ``cmd3`` are distinct subcommands of ``cmd1``.
+- :program:`--opt0` is an option that (potentially) affects **any** command, including the subcommands :program:`cmd2` or :program:`cmd3`.
+- :program:`--opt1` and :program:`arg1` are an option and mandatory argument to the command :program:`cmd1`.
+  They might not have any relevance to other :program:`mix-models` commands.
+- :program:`cmd2` and :program:`cmd3` are distinct subcommands of :program:`cmd1`.
 
-  - They *may* respond to ``--opt1`` and ``arg1``, and to ``--opt0``; at least, they *must* not contradict them.
+  - They *may* respond to :program:`--opt1` and :program:`arg1`, and to :program:`--opt0`; at least, they *must* not contradict them.
   - They each may have their own options and arguments, which can be distinct.
 
 .. tip:: Use ``--help`` for any (sub)command to read about its behaviour.
@@ -80,7 +93,7 @@ For some features of the code, the default behaviour is very elaborate and serve
 This default behaviour or optional behaviour is defined by reading an input file.
 These are stored in the :ref:`package data <package-data>` directory.
 
-For example, ``mix-models report`` loads reporting configuration from :file:`message_ix_models/data/report/global.yaml`, a YAML file with hundreds of lines.
+For example, :program:`mix-models report` loads reporting configuration from :file:`message_ix_models/data/report/global.yaml`, a YAML file with hundreds of lines.
 Optionally, a different file can be used::
 
     $ mix-models report --config other
@@ -99,77 +112,86 @@ Important CLI options and commands
 
 Top-level options and commands
 ------------------------------
-``mix-models --help`` describes these::
+:program:`mix-models --help` describes these::
 
-    $ mix-models --help
     Usage: mix-models [OPTIONS] COMMAND [ARGS]...
 
       Command-line interface for MESSAGEix-GLOBIOM model tools.
 
       Every tool and script in this repository is accessible through this CLI.
       Scripts are grouped into commands and sub-commands. For help on specific
-      (sub)commands, use --help, e.g.:
+      (sub)commands, use --help, for instance:
 
-              mix-models cd-links --help
-              mix-models cd-links run --help
+              mix-models report --help
+              mix-models ssp gen-structures --help
 
       The top-level options --platform, --model, and --scenario are used by
-      commands that access specific message_ix scenarios; these can also be
-      specified with --url.
+      commands that access specific MESSAGEix scenarios in a specific ixmp
+      platform/database; these can also be specified with --url.
 
-      For more information, see
-      https://docs.messageix.org/projects/models2/en/latest/cli.html
+      For complete documentation, see
+      https://docs.messageix.org/projects/models/en/latest/cli.html
 
     Options:
       --url ixmp://PLATFORM/MODEL/SCENARIO[#VERSION]
                                       Scenario URL.
-      --platform PLATFORM             Configured platform name.
+      --platform PLATFORM             ixmp platform name.
       --model MODEL                   Model name for some commands.
       --scenario SCENARIO             Scenario name for some commands.
-      --version INTEGER               Scenario version.
+      --version INTEGER               Scenario version for some commands.
       --local-data PATH               Base path for local data.
       -v, --verbose                   Print DEBUG-level log messages.
       --help                          Show this message and exit.
 
     Commands:
-      cd-links         CD-LINKS project.
-      dl               Retrieve data from primary sources.
-      engage           ENGAGE project.
-      iiasapp          Import power plant capacity.
-      material         Model with materials accounting.
-      prep-submission  Prepare scenarios for submission to database.
-      report           Postprocess results.
-      res              MESSAGE-GLOBIOM reference energy system (RES).
-      techs            Export data from data/technology.yaml to CSV.
-      transport        MESSAGEix-Transport variant.
-
-To explain further:
-
-``--platform PLATFORM`` or ``--url``
+      buildings         MESSAGEix-Buildings model.
+      cd-links          CD-LINKS project.
+      config            Get and set configuration keys.
+      covid             COVID project.
+      engage            ENGAGE project.
+      export-test-data  Prepare data for testing.
+      fetch             Retrieve data from primary sources.
+      iiasapp           Import power plant capacity.
+      last-log          Show the location of the last log file, if any.
+      material          Model with materials accounting.
+      model             MESSAGEix-GLOBIOM reference energy system (RES).
+      navigate          NAVIGATE project.
+      prep-submission   Prepare scenarios for submission to an IIASA Scenario...
+      report            Postprocess results.
+      res               MESSAGEix-GLOBIOM reference energy system (RES).
+      ssp               Shared Socioeconomic Pathways (SSP) project.
+      techs             Export metadata to technology.csv.
+      testing           Manipulate test data.
+      transport         MESSAGEix-Transport variant.
+      water-ix          MESSAGEix-Water and Nexus variant.
+
+Further information about the top-level options:
+
+:program:`--platform PLATFORM` or :program:`--url`
    By default, message_data connects to the default ixmp Platform.
    These options direct it to work with a different Platform.
 
-``--model MODEL --scenario SCENARIO`` or ``--url``
+:program:`--model MODEL --scenario SCENARIO` or :program:`--url`
     Many commands use an *existing* |Scenario| as a starting point, and begin by cloning that Scenario to a new (model name, scenario name).
     For any such command, these top-level options define the starting point/initial Scenario to clone/‘baseline’.
 
-    In contrast, see ``--output-model``, below.
-
+    In contrast, see :program:`--output-model`, below.
 
 Common options
 --------------
+
 Since :mod:`message_ix_models.model` and :mod:`message_ix_models.project` codes often perform similar tasks, their CLI options and arguments are provided in :mod:`.util.click` for easy re-use.
 These include:
 
-``ssp`` argument
+:program:`SSP` argument
    This takes one of the values 'SSP1', 'SSP2', or 'SSP3'.
 
    Commands that will not work for one or more of the SSPs should check the argument value given by the user and raise :class:`NotImplementedError`.
 
-``--output-model NAME`` option
-   This option is a counterpart to the top-level ``--url/--model/--scenario`` options.
+:program:`--output-model NAME` option
+   This option is a counterpart to the top-level :program:`--url`, :program:`--model`, or :program:`--scenario` options.
    A command that starts from one Scenario, and builds one or more Scenarios from it will clone *to* a new (model name, scenario name);
-   ``--output-model`` gives the model name.
+   :program:`--output-model` gives the model name.
 
    Current code generates a variety of fixed (non-configurable) scenario names; use ``--help`` for each command to see which.
 

diff --git a/doc/conf.py b/doc/conf.py
@@ -143,6 +143,7 @@ def local_inv(name: str, *parts: str) -> Optional[str]:
     ),
     "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
     "pint": ("https://pint.readthedocs.io/en/stable/", None),
+    "platformdirs": ("https://platformdirs.readthedocs.io/en/latest", None),
     "pooch": ("https://www.fatiando.org/pooch/latest/", None),
     "pytest": ("https://docs.pytest.org/en/stable/", None),
     "python": ("https://docs.python.org/3/", None),

diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst
@@ -4,6 +4,24 @@ What's new
 Next release
 ============
 
+- The :class:`.SSPUpdate` data provider pulls data from the SSP 2024 "Release 3.0" data files, and handles both the earlier and current structures (:pull:`156`).
+- Improve :class:`.ExoDataSource` with :meth:`.raise_on_extra_kw` utility method, automatic copy of source keyword arguments (:pull:`156`).
+- Expose :func:`.node.nodes_ex_world` for use as a genno (reporting) operator.
+- Raise DeprecationWarning from :func:`.util.sdmx.eval_anno`; remove internal usage of this deprecated method (:pull:`156`).
+- Reduce verbosity when using the :program:`mix-models` CLI when :mod:`message_data` is not installed (:issue:`37`, :pull:`156`).
+- Improve logging (:pull:`156`).
+
+  - Use multi-threaded logging for better performance.
+    Logging to stdout and file is on a separate thread and does not block operations on the main thread.
+  - Add automatic file logging.
+    Log versions of packages to file when using :func:`.workflow.make_click_command`.
+  - New CLI command :program:`mix-models last-log` to retrieve the location of the latest log file.
+- Update :doc:`cli` (:pull:`156`).
+- Improve performance in :func:`.disutility.data_conversion` (:pull:`156`).
+- Use :func:`platformdirs.user_cache_path` in more places; remove cache-path handling code (:pull:`156`).
+- Add :func:`.util.datetime_now_with_tz` (:pull:`156`).
+- Add :func:`.util.show_versions`, wrapping :func:`ixmp.util.show_versions` and returning its output as :class:`str` (:pull:`156`).
+- :func:`.util.private_data_path` returns an alternate, local data path if :mod:`message_data` is not installed (:pull:`156`).
 - Annotate :py:`c="transport"` in :ref:`the commodity code list <commodity-yaml>` with associated :ref:`IEA (E)WEB <tools-iea-web>` flows (:pull:`153`).
 
 v2024.1.29