Add complex filters for RunMetaEntries (#26)

* split meta.py into model and repo

* add __init__.py

* add filter

* add base filter

* adjust abstract signature and docs

* export meta base filter

* adjust api and api client

* adjust core api

* adjust tests

* add PATCH query endpoint

* fix bug for non-default run meta entries

* fix unused import

* Fix invalid-escape-sequences

* Add tests for meta-default-run filters

* Add tests for meta-key filters

---------

Co-authored-by: Fridolin Glatter <glatter@iiasa.ac.at>
Co-authored-by: Pino Mussak <mussak@iiasa.ac.at>

ixmp4/core/meta.py

@@ -1,16 +1,14 @@
-from typing import List
-
 import pandas as pd
 
 from .base import BaseFacade
 
 
 class MetaRepository(BaseFacade):
-    def tabulate(self, run_ids: List[int]) -> pd.DataFrame:
+    def tabulate(self, **kwargs) -> pd.DataFrame:
         # TODO: accept list of `Run` instances as arg
         # TODO: expand run-id to model-scenario-version-id columns
         return (
-            self.backend.meta.tabulate(run_ids=run_ids)
+            self.backend.meta.tabulate(**kwargs)
             .drop(columns=["id", "type"])
             .rename(columns={"run__id": "run_id"})
         )

ixmp4/core/run.py

@@ -105,7 +105,7 @@ def __init__(self, run: RunModel, *args, **kwargs) -> None:
         self.df, self.data = self._get()
 
     def _get(self) -> tuple[pd.DataFrame, dict]:
-        df = self.backend.meta.tabulate(run_ids=[self.run.id])
+        df = self.backend.meta.tabulate(run_id=self.run.id, run={"default_only": False})
         if df.empty:
             return df, {}
         return df, dict(zip(df["key"], df["value"]))

ixmp4/data/abstract/meta.py

@@ -126,18 +126,14 @@ def delete(self, id: int) -> None:
 
     def list(
         self,
-        *,
-        run_ids: Iterable[int] | None = None,
-        keys: Iterable[str] | None = None,
+        **kwargs,
     ) -> Iterable[RunMetaEntry]:
-        """Lists run's meta indicator entries by specified criteria.
+        r"""Lists run's meta indicator entries by specified criteria.
 
         Parameters
         ----------
-        run_ids : list of int
-            A list of run ids.
-        keys : list of str
-            A list of string keys.
+        \*\*kwargs: any
+            Filter parameters as specified in `ixmp4.data.db.meta.filter.RunMetaEntryFilter`.
 
         Returns
         -------
@@ -148,19 +144,14 @@ def list(
 
     def tabulate(
         self,
-        *,
-        run_ids: Iterable[int] | None = None,
-        keys: Iterable[str] | None = None,
+        **kwargs,
     ) -> pd.DataFrame:
-        """Tabulates run's meta indicator entries by specified criteria.
-
+        r"""Tabulates run's meta indicator entries by specified criteria.
 
         Parameters
         ----------
-        run_ids : list of int
-            A list of run ids.
-        keys : list of str
-            A list of string keys.
+        \*\*kwargs: any
+            Filter parameters as specified in `ixmp4.data.db.meta.filter.RunMetaEntryFilter`.
 
         Returns
         -------

ixmp4/data/abstract/optimization/indexset.py

@@ -79,14 +79,14 @@ def get(self, run_id: int, name: str) -> IndexSet:
         ...
 
     def list(self, *, name: str | None = None, **kwargs) -> Iterable[IndexSet]:
-        """Lists IndexSets by specified criteria.
+        r"""Lists IndexSets by specified criteria.
 
         Parameters
         ----------
         name : str
             The name of an IndexSet. If supplied only one result will be returned.
         # TODO: Update kwargs
-        **kwargs: any
+        \*\*kwargs: any
             More filter parameters as specified in
             `ixmp4.data.db.iamc.variable.filters.VariableFilter`.
 
@@ -98,14 +98,14 @@ def list(self, *, name: str | None = None, **kwargs) -> Iterable[IndexSet]:
         ...
 
     def tabulate(self, *, name: str | None = None, **kwargs) -> pd.DataFrame:
-        """Tabulate IndexSets by specified criteria.
+        r"""Tabulate IndexSets by specified criteria.
 
         Parameters
         ----------
         name : str
             The name of an IndexSet. If supplied only one result will be returned.
         # TODO: Update kwargs
-        **kwargs: any
+        \*\*kwargs: any
             More filter parameters as specified in
             `ixmp4.data.db.iamc.variable.filters.VariableFilter`.
 

ixmp4/data/api/meta.py

@@ -30,6 +30,7 @@ class RunMetaEntryRepository(
 ):
     model_class = RunMetaEntry
     prefix = "meta/"
+    enumeration_method = "PATCH"
 
     def create(
         self,
@@ -40,7 +41,7 @@ def create(
         return super().create(run__id=run__id, key=key, value=value)
 
     def get(self, run__id: int, key: str) -> RunMetaEntry:
-        return super().get(run__ids=[run__id], keys=[key])
+        return super().get(run_id=run__id, key=key)
 
     def delete(self, id: int) -> None:
         super().delete(id)

ixmp4/data/db/filters/__init__.py

@@ -1,3 +1,4 @@
+from .meta import RunMetaEntryFilter
 from .model import ModelFilter
 from .optimizationindexset import OptimizationIndexSetFilter
 from .region import RegionFilter

ixmp4/data/db/filters/meta.py

@@ -0,0 +1,23 @@
+from typing import ClassVar
+
+from ixmp4.db import filters
+
+from .. import RunMetaEntry
+
+
+class RunMetaEntryFilter(filters.BaseFilter, metaclass=filters.FilterMeta):
+    sqla_model: ClassVar[type] = RunMetaEntry
+
+    id: filters.Id
+    type: filters.String
+    run__id: filters.Integer = filters.Field(None, alias="run_id")
+
+    key: filters.String
+
+    value_int: filters.Integer
+    value_str: filters.String
+    value_float: filters.Float
+    value_bool: filters.Boolean
+
+    def join(self, exc, **kwargs):
+        return exc

ixmp4/data/db/meta/__init__.py

@@ -0,0 +1,2 @@
+from .model import RunMetaEntry
+from .repository import RunMetaEntryRepository

ixmp4/data/db/meta/filter.py

@@ -0,0 +1,19 @@
+from ixmp4.data.db import filters as base
+from ixmp4.data.db.run import Run
+from ixmp4.db import filters, utils
+
+from .model import RunMetaEntry
+
+
+class RunFilter(base.RunFilter, metaclass=filters.FilterMeta):
+    def join(self, exc, **kwargs):
+        if not utils.is_joined(exc, Run):
+            return exc.join(Run, onclause=RunMetaEntry.run__id == Run.id)
+        else:
+            return exc
+
+
+class RunMetaEntryFilter(base.RunMetaEntryFilter, metaclass=filters.FilterMeta):
+    run: RunFilter = filters.Field(
+        default=RunFilter(id=None, version=None, is_default=True)
+    )

ixmp4/data/db/meta/model.py

@@ -0,0 +1,76 @@
+from typing import ClassVar
+
+from ixmp4 import db
+from ixmp4.core.exceptions import InvalidRunMeta
+from ixmp4.data import abstract, types
+
+from .. import base
+
+
+class RunMetaEntry(base.BaseModel):
+    NotFound: ClassVar = abstract.RunMetaEntry.NotFound
+    NotUnique: ClassVar = abstract.RunMetaEntry.NotUnique
+    DeletionPrevented: ClassVar = abstract.RunMetaEntry.DeletionPrevented
+
+    Type: ClassVar = abstract.RunMetaEntry.Type
+
+    _column_map = {
+        abstract.RunMetaEntry.Type.INT: "value_int",
+        abstract.RunMetaEntry.Type.STR: "value_str",
+        abstract.RunMetaEntry.Type.FLOAT: "value_float",
+        abstract.RunMetaEntry.Type.BOOL: "value_bool",
+    }
+
+    __table_args__ = (
+        db.UniqueConstraint(
+            "run__id",
+            "key",
+        ),
+    )
+    updateable_columns = [
+        "type",
+        "value_int",
+        "value_str",
+        "value_float",
+        "value_bool",
+    ]
+
+    run__id: types.Integer = db.Column(
+        db.Integer,
+        db.ForeignKey("run.id"),
+        nullable=False,
+        index=True,
+    )
+    run = db.relationship(
+        "Run",
+        backref="meta",
+        foreign_keys=[run__id],
+    )
+
+    key: types.String = db.Column(db.String(1023), nullable=False)
+    type: types.String = db.Column(db.String(20), nullable=False)
+
+    value_int: types.Integer = db.Column(db.Integer, nullable=True)
+    value_str: types.String = db.Column(db.String(1023), nullable=True)
+    value_float: types.Float = db.Column(db.Float, nullable=True)
+    value_bool: types.Boolean = db.Column(db.Boolean, nullable=True)
+
+    @property
+    def value(self) -> abstract.MetaValue:
+        type_ = RunMetaEntry.Type(self.type)
+        col = self._column_map[type_]
+        return getattr(self, col)
+
+    def __init__(self, *args, **kwargs) -> None:
+        value = kwargs.pop("value")
+        value_type = type(value)
+        try:
+            type_ = RunMetaEntry.Type.from_pytype(value_type)
+            col = self._column_map[type_]
+        except KeyError:
+            raise InvalidRunMeta(
+                f"Invalid type `{value_type}` for value of `RunMetaEntry`."
+            )
+        kwargs["type"] = type_
+        kwargs[col] = value
+        super().__init__(*args, **kwargs)