Implement DIGSY scenarios part2

.github/workflows/transport.yaml

@@ -26,20 +26,7 @@ env:
   # - Ensure there is NO trailing comma on the last line.
   labels: >-
     [
-      "SSP1",
-      "SSP1 policy",
-      "SSP2",
-      "SSP2 policy",
-      "SSP3",
-      "SSP3 policy",
-      "SSP4",
-      "SSP4 policy",
-      "SSP5",
-      "SSP5 policy",
-      "EDITS-CA",
-      "EDITS-HA",
-      "LED-SSP1",
-      "LED-SSP2"
+     "LED-SSP2"
     ]
 
   # Currently disabled:
@@ -49,8 +36,8 @@ env:
 
 on:
   # Uncomment these lines for debugging, but leave them commented on 'main'
-  # pull_request:
-  #   branches: [ main ]
+  pull_request:
+    branches: [ main ]
   # push:
   #   branches: [ main ]
   schedule:

doc/api/tools.rst

@@ -30,70 +30,76 @@ Exogenous data (:mod:`.tools.exo_data`)
    :members:
    :exclude-members: ExoDataSource, prepare_computer
 
+   The tools in this module support use of data from arbitrary sources and formats in model-building code.
+   For each source/format, a subclass of :class:`.ExoDataSource` adds tasks to a :class:`genno.Computer`
+   that retrieve/load and transform the source data into :class:`genno.Quantity`.
+
+   An example using one such class, :class:`message_ix_models.project.advance.data.ADVANCE`.
+
+   .. code-block:: python
+
+      from genno import Computer
+
+      from message_ix_models.project.advance.data import ADVANCE
+
+      # Keyword arguments corresponding to ADVANCE.Options
+      kw = dict(
+          measure="Transport|Service demand|Road|Passenger|LDV",
+          model="MESSAGE",
+          scenario="ADV3TRAr2_Base",
+      )
+
+      # Add tasks to retrieve and transform data
+      c = Computer()
+      keys = c.apply(ADVANCE, context=context, **kw)
+
+      # Retrieve some of the data
+      q_result = c.get(keys[0])
+
+      # Pass the data into further calculations
+      c.add("derived", "mul", keys[1], k_other)
+
    .. autosummary::
 
       MEASURES
       SOURCES
+      BaseOptions
       DemoSource
       ExoDataSource
+      add_structure
       iamc_like_data_for_query
       prepare_computer
       register_source
 
-.. autofunction:: prepare_computer
-
-   The first returned key, like ``{measure}:n-y``, triggers the following computations:
-
-   1. Load data by invoking a :class:`ExoDataSource`.
-   2. Aggregate on the |n| (node) dimension according to :attr:`.Config.regions`.
-   3. Interpolate on the |y| (year) dimension according to :attr:`.Config.years`.
-
-   Additional key(s) include:
-
-   - ``{measure}:n-y:y0 indexed``: same as ``{measure}:n-y``, indexed to values as of |y0| (the first model year).
-
-   See particular data source classes, like :class:`.SSPOriginal`, for particular examples of usage.
-
-   .. todo:: Extend to also prepare to compute values indexed to a particular |n|.
-
 .. autoclass:: ExoDataSource
    :members:
    :private-members: _where
-   :special-members: __init__, __call__
-
-.. currentmodule:: message_ix_models.tools.advance
+   :special-members: __init__
 
-ADVANCE data (:mod:`.tools.advance`)
-====================================
-
-.. deprecated:: 2023.11
-   Use :mod:`.project.advance` instead.
-
-.. autosummary::
-   get_advance_data
-   advance_data
+   As an abstract class ExoDataSource **must** be subclassed to be used.
+   Concrete subclasses **must** implement at least the :meth:`~ExoDataSource.get` method
+   that performs the loading of the raw data when executed,
+   and **may** override others, as described below.
 
-.. autodata:: LOCATION
+   The class method :meth:`.ExoDataSource.add_tasks` adds tasks to a :class:`genno.Computer`.
+   It returns a :class:`genno.Key` that refers to the loaded and transformed data.
+   This method usually **should not** be modified for subclasses.
 
-This is a location relative to a parent directory.
-The specific parent directory depends on whether :mod:`message_data` is available:
+   The behaviour of a subclass can be customized in these ways:
 
-Without :mod:`message_data`:
-   The code finds the data within :ref:`local-data` (see discussion there for how to configure this location).
-   Users should:
+   1. Create a subclass of :class:`.BaseOptions`
+      and set it as the :attr:`~.ExoDataSource.Options` class attribute.
+   2. Override :meth:`~.ExoDataSource.__init__`,
+      which receives keyword arguments via :meth:`.add_tasks`.
+   3. Override :meth:`~.ExoDataSource.transform`,
+      which is called to add further tasks which will transform the data.
 
-   1. Visit https://tntcat.iiasa.ac.at/ADVANCEWP2DB/dsd?Action=htmlpage&page=about and register for access to the data.
-   2. Log in.
-   3. Download the snapshot with the file name given in :data:`LOCATION` to a subdirectory :file:`advance/` within their local data directory.
+   See the documentation for these methods and attributes for further details.
 
-With :mod:`message_data`:
-   The code finds the data within :ref:`private-data`.
-   The snapshot is stored directly in the repository using Git LFS.
+.. autofunction:: prepare_computer
 
-.. automodule:: message_ix_models.tools.advance
-   :members:
-   :exclude-members: LOCATION
-   :private-members:
+   .. deprecated:: 2025-06-06
+      Use :py:`c.apply(SOURCE.add_tasks, …)` as shown above.
 
 .. currentmodule:: message_ix_models.tools.iamc
 
@@ -103,6 +109,14 @@ IAMC data structures (:mod:`.tools.iamc`)
 .. automodule:: message_ix_models.tools.iamc
    :members:
 
+.. currentmodule:: message_ix_models.tools.policy
+
+Policies (:mod:`.tools.policy`)
+===============================
+
+.. automodule:: message_ix_models.tools.policy
+   :members:
+
 .. _tools-wb:
 
 World Bank structures (:mod:`.tools.wb`)

message_ix_models/data/emission.yaml

@@ -39,6 +39,10 @@ SO2:
   name: Sulfur dioxide
   report: Sulfur
 
+TCE:
+  name: (missing)
+  type: TCE
+
 VOC:
   name: Volatile organic compounds
 
@@ -79,7 +83,6 @@ VOC:
 # SF6
 # SO2_LandUseChangeEM
 # SO2_SavanBurnEM
-# TCE
 # TCE_CO2
 # TCE_non-CO2
 # VOC_LandUseChangeEM

message_ix_models/data/sdmx/IIASA_ECE_AGENCIES(0.1).xml

@@ -1,14 +1,17 @@
-<mes:Structure xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:com="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/common" xmlns:data="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/data/structurespecific" xmlns:str="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/structure" xmlns:mes="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message" xmlns:gen="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/data/generic" xmlns:footer="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message/footer">
+<?xml version='1.0' encoding='utf-8'?>
+<mes:Structure xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:com="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/common" xmlns:md="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/metadata/generic" xmlns:md_ss="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/metadata/structurespecific" xmlns:data="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/data/structurespecific" xmlns:str="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/structure" xmlns:mes="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message" xmlns:gen="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/data/generic" xmlns:footer="http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message/footer">
   <mes:Header>
+    <mes:ID>none</mes:ID>
     <mes:Test>false</mes:Test>
-    <mes:Prepared>2023-09-04T16:31:44.700655</mes:Prepared>
-    <mes:Source xml:lang="en">Generated by message_ix_models 2023.5.32.dev20+g8d51636</mes:Source>
+    <mes:Prepared>2025-05-19T22:09:29.424849</mes:Prepared>
+    <mes:Sender id="none"/>
+    <mes:Source xml:lang="en">Generated by message_ix_models 2025.1.11.dev398+gdaf97af4a.d20250504</mes:Source>
   </mes:Header>
   <mes:Structures>
     <str:OrganisationSchemes>
-      <str:AgencyScheme version="0.1" isExternalReference="false" isFinal="false" agencyID="IIASA_ECE" id="AGENCIES" urn="urn:sdmx:org.sdmx.infomodel.base.AgencyScheme=IIASA_ECE:AGENCIES(0.1)">
+      <str:AgencyScheme isExternalReference="false" isFinal="false" agencyID="IIASA_ECE" version="0.1" id="AGENCIES" urn="urn:sdmx:org.sdmx.infomodel.base.AgencyScheme=IIASA_ECE:AGENCIES(0.1)">
         <com:Description xml:lang="en">Agencies referenced by data structures in message_ix_models</com:Description>
-        <str:Agency id="IIASA_ECE">
+        <str:Agency id="IIASA_ECE" urn="urn:sdmx:org.sdmx.infomodel.base.Agency=IIASA_ECE:AGENCIES(0.1).IIASA_ECE">
           <com:Name xml:lang="en">IIASA Energy, Climate, and Environment Program</com:Name>
         </str:Agency>
         <str:Agency id="ICONICS">
@@ -17,7 +20,7 @@
             <str:URI>https://depts.washington.edu/iconics/</str:URI>
           </str:Contact>
         </str:Agency>
-        <str:Agency id="IEA">
+        <str:Agency id="IEA" urn="urn:sdmx:org.sdmx.infomodel.base.Agency=IIASA_ECE:AGENCIES(0.1).IEA">
           <com:Name xml:lang="en">International Energy Agency</com:Name>
           <str:Contact>
             <str:URI>https://iea.org</str:URI>