Merge 977fad2 into e15b67b

flexmeasures/data/models/planning/storage.py

@@ -191,6 +191,35 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
                 sensor=inflexible_sensor,
             )
 
+        # fetch SOC constraints from sensors
+        if isinstance(soc_targets, Sensor):
+            soc_targets = get_continuous_series_sensor_or_quantity(
+                quantity_or_sensor=soc_targets,
+                actuator=sensor,
+                unit="MWh",
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+            )
+        if isinstance(soc_minima, Sensor):
+            soc_minima = get_continuous_series_sensor_or_quantity(
+                quantity_or_sensor=soc_minima,
+                actuator=sensor,
+                unit="MWh",
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+            )
+        if isinstance(soc_maxima, Sensor):
+            soc_maxima = get_continuous_series_sensor_or_quantity(
+                quantity_or_sensor=soc_maxima,
+                actuator=sensor,
+                unit="MWh",
+                query_window=(start, end),
+                resolution=resolution,
+                beliefs_before=belief_time,
+            )
+
         device_constraints[0] = add_storage_constraints(
             start,
             end,
@@ -500,7 +529,8 @@ def possibly_extend_end(self):
               this function would become a class method with a @post_load decorator.
         """
         soc_targets = self.flex_model.get("soc_targets")
-        if soc_targets:
+
+        if soc_targets and not isinstance(soc_targets, Sensor):
             max_target_datetime = max([soc_target["end"] for soc_target in soc_targets])
             if max_target_datetime > self.end:
                 max_server_horizon = get_max_planning_horizon(self.resolution)
@@ -515,6 +545,11 @@ def get_min_max_targets(
         min_target = None
         max_target = None
         soc_targets_label = "soc_targets" if deserialized_names else "soc-targets"
+
+        # if the SOC targets are defined as a Sensor, we don't get min max values
+        if isinstance(self.flex_model.get(soc_targets_label), dict):
+            return None, None
+
         if (
             soc_targets_label in self.flex_model
             and len(self.flex_model[soc_targets_label]) > 0

flexmeasures/data/models/planning/tests/conftest.py

@@ -368,6 +368,50 @@ def add_storage_efficiency(db, add_battery_assets, setup_sources) -> dict[str, S
     return sensors
 
 
+@pytest.fixture(scope="module")
+def add_soc_targets(db, add_battery_assets, setup_sources) -> dict[str, Sensor]:
+    """
+    Fixture to add storage SOC targets as sensors and their beliefs to the database.
+
+    The function creates a single event at 14:00 + offset with a value of 0.5.
+    """
+
+    battery = add_battery_assets["Test battery"]
+    soc_value = 0.5
+    soc_datetime = pd.Timestamp("2015-01-01T14:00:00", tz="Europe/Amsterdam")
+    sensors = {}
+
+    sensor_specs = [
+        # name, resolution, offset from the resolution tick
+        ("soc-targets (1h)", timedelta(minutes=60), timedelta(minutes=0)),
+        ("soc-targets (15min)", timedelta(minutes=15), timedelta(minutes=0)),
+        ("soc-targets (15min lagged)", timedelta(minutes=15), timedelta(minutes=5)),
+        ("soc-targets (instantaneous)", timedelta(minutes=0), timedelta(minutes=0)),
+    ]
+
+    for name, resolution, offset in sensor_specs:
+        storage_constraint_sensor = Sensor(
+            name=name,
+            unit="MWh",
+            event_resolution=resolution,
+            generic_asset=battery,
+        )
+        db.session.add(storage_constraint_sensor)
+        db.session.flush()
+
+        belief = TimedBelief(
+            event_start=soc_datetime + offset,
+            belief_horizon=timedelta(hours=100),
+            event_value=soc_value,
+            source=setup_sources["Seita"],
+            sensor=storage_constraint_sensor,
+        )
+        db.session.add(belief)
+        sensors[name] = storage_constraint_sensor
+
+    return sensors
+
+
 def add_as_beliefs(db, sensor, values, time_slots, source):
     beliefs = [
         TimedBelief(

flexmeasures/data/models/planning/tests/test_solver.py

@@ -1790,3 +1790,86 @@ def test_battery_storage_efficiency_sensor(
 
     scheduler_info = scheduler._prepare()
     assert all(scheduler_info[5][0]["efficiency"] == expected_efficiency)
+
+
+@pytest.mark.parametrize(
+    "sensor_name, expected_start, expected_end",
+    [
+        (
+            "soc-targets (1h)",
+            "14:00:00",
+            "14:45:00",
+        ),  # A value defined in a larger resolution is downsampled to match the power sensor resolution.
+        (
+            "soc-targets (15min)",
+            "14:00:00",
+            "14:00:00",
+        ),  # A simple case, SOC constraint sensor in the sample resolution as the power sensor.
+        (
+            "soc-targets (instantaneous)",
+            "14:00:00",
+            "14:00:00",
+        ),  # For an instantaneous sensor, the value is set to the interval containing the instantaneous event.
+        pytest.param(  # This is an event at 14:05:00 with a duration of 15min. These constraint should span the intervals 14:00 and 14:15 but we are not reindexing properly.
+            "soc-targets (15min lagged)",
+            "14:00:00",
+            "14:15:00",
+            marks=pytest.mark.xfail(
+                reason="we should re-index the series so that values of the original index that overlap are used."
+            ),
+        ),
+    ],
+)
+def test_add_storage_constraint_from_sensor(
+    add_battery_assets,
+    add_soc_targets,
+    sensor_name,
+    expected_start,
+    expected_end,
+    db,
+):
+    """
+    Test the handling of different values for the target SOC constraints as sensors in the StorageScheduler.
+    """
+    _, battery = get_sensors_from_db(db, add_battery_assets)
+    tz = pytz.timezone("Europe/Amsterdam")
+    start = tz.localize(datetime(2015, 1, 1))
+    end = tz.localize(datetime(2015, 1, 2))
+    resolution = timedelta(minutes=15)
+    soc_targets = add_soc_targets[sensor_name]
+
+    flex_model = {
+        "soc-max": 2,
+        "soc-min": 0,
+        "roundtrip-efficiency": 1,
+        "production-capacity": "0kW",
+        "soc-at-start": 0,
+    }
+
+    flex_model["soc-targets"] = {"sensor": soc_targets.id}
+
+    scheduler: Scheduler = StorageScheduler(
+        battery, start, end, resolution, flex_model=flex_model
+    )
+
+    scheduler_info = scheduler._prepare()
+    storage_constraints = scheduler_info[5][0]
+
+    expected_target_start = pd.Timedelta(expected_start) + start
+    expected_target_end = pd.Timedelta(expected_end) + start
+    expected_soc_target_value = 0.5 * timedelta(hours=1) / resolution
+
+    # convert dates from UTC to local time (Europe/Amsterdam)
+    equals = storage_constraints["equals"].tz_convert(tz)
+
+    # check that no value before expected_target_start is non-nan
+    assert all(equals[: expected_target_start - resolution].isna())
+
+    # check that no value after expected_target_end is non-nan
+    assert all(equals[expected_target_end + resolution :].isna())
+
+    # check that the values in the (expected_target_start, expected_target_end) are equal to the expected value
+    assert all(
+        equals[expected_target_start + resolution : expected_target_end]
+        == expected_soc_target_value
+    )

flexmeasures/data/schemas/scheduling/storage.py

@@ -15,12 +15,19 @@
 from marshmallow.validate import OneOf, ValidationError, Validator
 import pandas as pd
 
+from flexmeasures.data import db
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.schemas.times import AwareDateTimeField, DurationField
 from flexmeasures.data.schemas.units import QuantityField
 from flexmeasures.data.schemas.sensors import QuantityOrSensor
 
-from flexmeasures.utils.unit_utils import ur
+from flexmeasures.utils.unit_utils import ur, units_are_convertible
+
+from flexmeasures.data.schemas.utils import (
+    FMValidationError,
+    MarshmallowClickMixin,
+    with_appcontext_if_needed,
+)
 
 
 class EfficiencyField(QuantityField):
@@ -128,6 +135,57 @@ def check_time_window(self, data: dict, **kwargs):
             data["end"] = end
 
 
+class TimeSeriesOrSensor(MarshmallowClickMixin, fields.Field):
+    def __init__(
+        self, unit, timezone, *args, value_validator: Validator | None = None, **kwargs
+    ):
+        super().__init__(*args, **kwargs)
+        self.timezone = timezone
+        self.value_validator = value_validator
+        self.unit = ur.Quantity(unit)
+
+    @with_appcontext_if_needed()
+    def _deserialize(
+        self, value: str | dict[str, int], attr, obj, **kwargs
+    ) -> list[dict] | Sensor:
+
+        if isinstance(value, dict):
+            if "sensor" not in value:
+                raise FMValidationError(
+                    "Dictionary provided but `sensor` key not found."
+                )
+
+            sensor = db.session.get(Sensor, value["sensor"])
+
+            if sensor is None:
+                raise FMValidationError(f"No sensor found with id {value['sensor']}.")
+
+            # lazy loading now (sensor is somehow not in session after this)
+            sensor.generic_asset
+            sensor.generic_asset.generic_asset_type
+
+            if not units_are_convertible(sensor.unit, str(self.unit.units)):
+                raise FMValidationError(
+                    f"Cannot convert {sensor.unit} to {self.unit.units}"
+                )
+            return sensor
+
+        elif isinstance(value, list):
+            field = fields.List(
+                fields.Nested(
+                    SOCValueSchema(
+                        timezone=self.timezone, value_validator=self.value_validator
+                    )
+                )
+            )
+
+            return field._deserialize(value, None, None)
+        else:
+            raise FMValidationError(
+                f"Unsupported value type. `{type(value)}` was provided but only dict and list are supported."
+            )
+
+
 class StorageFlexModelSchema(Schema):
     """
     This schema lists fields we require when scheduling storage assets.
@@ -152,21 +210,19 @@ class StorageFlexModelSchema(Schema):
     )
 
     # Timezone placeholder is overridden in __init__
-    soc_maxima = fields.List(
-        fields.Nested(SOCValueSchema(timezone="placeholder")),
-        data_key="soc-maxima",
+    soc_maxima = TimeSeriesOrSensor(
+        unit="MWh", timezone="placeholder", data_key="soc-maxima"
     )
-    soc_minima = fields.List(
-        fields.Nested(
-            SOCValueSchema(
-                timezone="placeholder", value_validator=validate.Range(min=0)
-            )
-        ),
+
+    soc_minima = TimeSeriesOrSensor(
+        unit="MWh",
+        timezone="placeholder",
         data_key="soc-minima",
+        value_validator=validate.Range(min=0),
     )
-    soc_targets = fields.List(
-        fields.Nested(SOCValueSchema(timezone="placeholder")),
-        data_key="soc-targets",
+
+    soc_targets = TimeSeriesOrSensor(
+        unit="MWh", timezone="placeholder", data_key="soc-targets"
     )
 
     soc_unit = fields.Str(
@@ -204,28 +260,27 @@ def __init__(self, start: datetime, sensor: Sensor, *args, **kwargs):
         """Pass the schedule's start, so we can use it to validate soc-target datetimes."""
         self.start = start
         self.sensor = sensor
-        self.soc_maxima = fields.List(
-            fields.Nested(SOCValueSchema(timezone=sensor.timezone)),
-            data_key="soc-maxima",
+        self.soc_maxima = TimeSeriesOrSensor(
+            unit="MWh", timezone=sensor.timezone, data_key="soc-maxima"
         )
-        self.soc_minima = fields.List(
-            fields.Nested(
-                SOCValueSchema(
-                    timezone=sensor.timezone, value_validator=validate.Range(min=0)
-                )
-            ),
+
+        self.soc_minima = TimeSeriesOrSensor(
+            unit="MWh",
+            timezone=sensor.timezone,
             data_key="soc-minima",
+            value_validator=validate.Range(min=0),
         )
-        self.soc_targets = fields.List(
-            fields.Nested(SOCValueSchema(timezone=sensor.timezone)),
-            data_key="soc-targets",
+        self.soc_targets = TimeSeriesOrSensor(
+            unit="MWh", timezone=sensor.timezone, data_key="soc-targets"
         )
+
         super().__init__(*args, **kwargs)
 
     @validates_schema
     def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs):
         soc_targets: list[dict[str, datetime | float]] | None = data.get("soc_targets")
-        if not soc_targets:
+        # skip check if the SOC targets are not provided or if they are defined as sensors
+        if not soc_targets or isinstance(soc_targets, Sensor):
             return
         max_server_horizon = current_app.config.get("FLEXMEASURES_MAX_PLANNING_HORIZON")
         if isinstance(max_server_horizon, int):