Fix CI, tests, docs build of the plugin-branch

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.10, 3.11]
+        python-version: ["3.10", "3.11"]
       fail-fast: false
     name: Run tests in conda environment ${{ matrix.python-version }}
     steps:

.github/workflows/sonar.yml

@@ -19,7 +19,7 @@ jobs:
           activate-environment: ewatercycle
           environment-file: environment.yml
           mamba-version: "*"
-          python-version: 3.10
+          python-version: "3.10"
           miniconda-version: "latest"
           channels: conda-forge
       - name: Install dependencies

CHANGELOG.md

@@ -16,11 +16,16 @@ Formatted as described on [https://keepachangelog.com](https://keepachangelog.co
     - Model container images using BMI v0.2 are supported see [grpc4bmi docs](https://grpc4bmi.readthedocs.io/en/latest/server/python.html#legacy-version).
 - ewatercycle config, forcings and parameter sets now use Pydantic for validation instead of Matplotlib inspired validation. ([#332](https://github.com/eWaterCycle/ewatercycle/issues/332), [#334](https://github.com/eWaterCycle/ewatercycle/pull/334), [#346](https://github.com/eWaterCycle/ewatercycle/pull/346))
 - Functions of a model inside a container that return the same result each call are cached with [MemoizedBmi](https://grpc4bmi.readthedocs.io/en/latest/api/grpc4bmi.bmi_memoized.html#grpc4bmi.bmi_memoized.MemoizedBmi) ([#339](https://github.com/eWaterCycle/ewatercycle/pull/339))
+- Moved CaseConfig to src/utils.py
+- forcing.load_foreign has been superceded by using sources.model(...)
 
 ### Deprecated
 
 - Singularity support ([#290](https://github.com/eWaterCycle/ewatercycle/issues/290))
 
+## Removed
+- Removed parametersetdb module. XmlConfig moved to lisflood plugin. YamlConfig & IniConfig have been removed.
+
 ## [1.4.1] (2022-12-20)
 
 ### Fixed

environment.yml

@@ -3,8 +3,7 @@ name: ewatercycle
 channels:
   - conda-forge
 dependencies:
-  # Numba 0.56.4 does not support Python 3.11 so set upper limit
-  - python>=3.10,<3.11
+  - python>=3.10,<3.12
   - esmvaltool-python>=2.3.0
   - subversion
   # Pin esmpy so we dont get forced to run all parallel tasks on single cpu

src/ewatercycle/forcing/_default.py → src/ewatercycle/base/forcing.py

@@ -1,17 +1,17 @@
-"""Forcing related functionality for default models."""
-import logging
-from pathlib import Path
-from typing import Literal, Optional, Union
-
 from esmvalcore.experimental import CFG
-from esmvalcore.experimental.config import Session
+from esmvalcore.config import Session
+from ewatercycle.util import to_absolute_path
+
+import logging
 from pydantic import BaseModel, validator
 from ruamel.yaml import YAML
 
-from ewatercycle.util import to_absolute_path
 
-logger = logging.getLogger(__name__)
+from pathlib import Path
+from typing import Literal, Optional, Union
+
 
+logger = logging.getLogger(__name__)
 FORCING_YAML = "ewatercycle_forcing.yaml"
 
 
@@ -80,6 +80,43 @@ def save(self):
         with open(target, "w") as f:
             yaml.dump(fdict, f)
         return target
+
+    @classmethod
+    def load(cls, directory: str | Path):
+        """Load previously generated or imported forcing data.
+
+        Args:
+            directory: forcing data directory; must contain
+                `ewatercycle_forcing.yaml` file
+
+        Returns: Forcing object
+        """
+        data_source = to_absolute_path(directory)
+        meta = (data_source / FORCING_YAML)
+        yaml = YAML(typ="safe")
+
+        if not meta.exists():
+            raise FileNotFoundError(
+                f"Forcing file {meta} not found. "
+                f"Perhaps you want to use {cls.__name__}(...)?"
+            )
+        metadata = meta.read_text()
+        # Workaround for legacy forcing files having !PythonClass tag.
+        #     Get model name of non-initialized BaseModel with Pydantic class property:
+        modelname = cls.__fields__['model'].default
+        metadata = metadata.replace(
+            f"!{cls.__name__}",
+            f"model: {modelname}"
+        )
+
+        fdict = yaml.load(metadata)
+        fdict["directory"] = data_source
+
+        return cls(**fdict)
+
+    @classmethod
+
+
 
     def plot(self):
         raise NotImplementedError("No generic plotting method available.")
@@ -103,3 +140,21 @@ def session_dir(self):
             return self.output_dir
 
     return TimeLessSession(Path(directory).absolute())
+
+
+DATASETS = {
+    "ERA5": {
+        "dataset": "ERA5",
+        "project": "OBS6",
+        "tier": 3,
+        "type": "reanaly",
+        "version": 1,
+    },
+    "ERA-Interim": {
+        "dataset": "ERA-Interim",
+        "project": "OBS6",
+        "tier": 3,
+        "type": "reanaly",
+        "version": 1,
+    },
+}

src/ewatercycle/model.py → src/ewatercycle/base/model.py

@@ -1,19 +1,22 @@
-"""Abstract class of a eWaterCycle model."""
-import logging
-from abc import ABCMeta, abstractmethod
-from datetime import datetime
-from typing import Any, ClassVar, Generic, Iterable, Optional, Tuple, TypeVar
+from ewatercycle._repr import Representation
+from ewatercycle.base.parameter_set import ParameterSet
+from ewatercycle.base.forcing import DefaultForcing
 
+
+import logging
 import numpy as np
 import xarray as xr
+from typing import TypeVar
 from bmipy import Bmi
 from cftime import num2date
 from grpc4bmi.bmi_optionaldest import OptionalDestBmi
 from grpc4bmi.reserve import reserve_values, reserve_values_at_indices
 
-from ewatercycle._repr import Representation
-from ewatercycle.forcing import DefaultForcing
-from ewatercycle.base.parameter_set import ParameterSet
+
+from abc import ABCMeta, abstractmethod
+from datetime import datetime
+from typing import Any, ClassVar, Generic, Iterable, Optional, Tuple
+
 
 logger = logging.getLogger(__name__)
 
@@ -57,24 +60,19 @@ def __repr_args__(self):
     @abstractmethod
     def setup(self, *args, **kwargs) -> Tuple[str, str]:
         """Performs model setup.
-
         1. Creates config file and config directory
         2. Start bmi container and store as self.bmi
-
         Args:
             *args: Positional arguments. Sub class should specify each arg.
             **kwargs: Named arguments. Sub class should specify each arg.
-
         Returns:
             Path to config file and path to config directory
         """
 
     def initialize(self, config_file: str) -> None:
         """Initialize the model.
-
         Args:
             config_file: Name of initialization file.
-
         """
         self.bmi.initialize(config_file)
 
@@ -89,10 +87,8 @@ def update(self) -> None:
 
     def get_value(self, name: str) -> np.ndarray:
         """Get a copy of values of the given variable.
-
         Args:
             name: Name of variable
-
         """
         if isinstance(self.bmi, OptionalDestBmi):
             return self.bmi.get_value(name)
@@ -103,12 +99,10 @@ def get_value_at_coords(
         self, name, lat: Iterable[float], lon: Iterable[float]
     ) -> np.ndarray:
         """Get a copy of values of the given variable at lat/lon coordinates.
-
         Args:
             name: Name of variable
             lat: Latitudinal value
             lon: Longitudinal value
-
         """
         indices = self._coords_to_indices(name, lat, lon)
         indices = np.array(indices)
@@ -119,25 +113,21 @@ def get_value_at_coords(
 
     def set_value(self, name: str, value: np.ndarray) -> None:
         """Specify a new value for a model variable.
-
         Args:
             name: Name of variable
             value: The new value for the specified variable.
-
         """
         self.bmi.set_value(name, value)
 
     def set_value_at_coords(
         self, name: str, lat: Iterable[float], lon: Iterable[float], values: np.ndarray
     ) -> None:
         """Specify a new value for a model variable at at lat/lon coordinates.
-
         Args:
             name: Name of variable
             lat: Latitudinal value
             lon: Longitudinal value
             values: The new value for the specified variable.
-
         """
         indices = self._coords_to_indices(name, lat, lon)
         indices = np.array(indices)
@@ -147,11 +137,9 @@ def _coords_to_indices(
         self, name: str, lat: Iterable[float], lon: Iterable[float]
     ) -> Iterable[int]:
         """Converts lat/lon values to index.
-
         Args:
             lat: Latitudinal value
             lon: Longitudinal value
-
         """
         raise NotImplementedError(
             "Method to convert from coordinates to model indices "
@@ -161,12 +149,9 @@ def _coords_to_indices(
     @abstractmethod
     def get_value_as_xarray(self, name: str) -> xr.DataArray:
         """Get a copy values of the given variable as xarray DataArray.
-
         The xarray object also contains coordinate information and additional
         attributes such as the units.
-
         Args: name: Name of the variable
-
         """
 
     @property
@@ -207,23 +192,20 @@ def output_var_names(self) -> Iterable[str]:
     @property
     def start_time_as_isostr(self) -> str:
         """Start time of the model.
-
         In UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'.
         """
         return self.start_time_as_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")
 
     @property
     def end_time_as_isostr(self) -> str:
         """End time of the model.
-
         In UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'.
         """
         return self.end_time_as_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")
 
     @property
     def time_as_isostr(self) -> str:
         """Current time of the model.
-
         In UTC and ISO format string e.g. 'YYYY-MM-DDTHH:MM:SSZ'.
         """
         return self.time_as_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -283,4 +265,4 @@ def _check_version(self):
             raise ValueError(
                 f"Supplied version {self.version} is not supported by this model. "
                 f"Available versions are {self.available_versions}."
-            )
+            )

src/ewatercycle/forcing.py

@@ -0,0 +1,50 @@
+"""Forcing module of eWaterCycle. Contains the model sources."""
+from importlib.metadata import entry_points
+from importlib_metadata import EntryPoint
+from collections.abc import Mapping
+from ewatercycle.base.forcing import DefaultForcing
+from typing import Type, Any
+
+
+class ForcingSources(Mapping):
+    """Lazy dictionary to hold the different forcing sources.
+
+    The `sources` object holds the forcing sources.
+    Forcing can be generated for a specifc source by doing, for example:
+        `from ewatercycle.forcing import sources`,
+        `forcing = sources.MarrmotForcing.generate(...)`
+    """
+    def __init__(self, *args, **kw):
+        self._raw_dict = dict(*args, **kw)
+
+    def __getitem__(self, key) -> Type[DefaultForcing]:
+        """Gets the entry point, loads it, and returns the Forcing object."""
+        if isinstance(self._raw_dict[key], EntryPoint):
+            return self._raw_dict[key].load()
+        else:
+            return self._raw_dict[key]
+
+    def __getattr__(self, attr):
+        """Accesses the keys like attributes. E.g. sources.HypeForcing."""
+        if attr in self._raw_dict.keys():
+            return self.__getitem__(attr)
+        else:
+            return getattr(self._raw_dict, attr)
+
+    def __iter__(self):
+        return iter(self._raw_dict)
+
+    def __len__(self):
+        return len(self._raw_dict)
+
+    def __repr__(self):
+        return self.__class__.__name__ + str(list(self._raw_dict.keys()))
+
+
+_forcings: dict[str, Any] = {
+    entry_point.name: entry_point
+    for entry_point in entry_points(group="ewatercycle.forcings")
+}
+_forcings["DefaultForcing"] = DefaultForcing
+
+sources = ForcingSources(_forcings)