From 49a155a62ed5b01f032d2c6f99ea940d1bcf6b68 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 23 Nov 2023 20:27:05 +0100 Subject: [PATCH 1/2] fix: separate logic for falling back on a default attribute and applying a maximum capacity limit Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 10 ++--- .../data/models/planning/tests/test_solver.py | 2 +- flexmeasures/data/models/planning/utils.py | 43 ++++++++----------- 3 files changed, 22 insertions(+), 33 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 53139a66d..a8e570830 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -214,9 +214,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 query_window=(start, end), resolution=resolution, beliefs_before=belief_time, - default_value_attribute="production_capacity", - default_value=convert_units(power_capacity_in_mw, "MW", sensor.unit), - method="upper", + fallback_attribute="production_capacity", + max_value=convert_units(power_capacity_in_mw, "MW", sensor.unit), ) if sensor.get_attribute("is_strictly_non_negative"): device_constraints[0]["derivative max"] = 0 @@ -230,9 +229,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 query_window=(start, end), resolution=resolution, beliefs_before=belief_time, - default_value_attribute="consumption_capacity", - default_value=convert_units(power_capacity_in_mw, "MW", sensor.unit), - method="upper", + fallback_attribute="consumption_capacity", + max_value=convert_units(power_capacity_in_mw, "MW", sensor.unit), ) # Apply round-trip efficiency evenly to charging and discharging diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 24a593649..c50f06dbf 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -1305,7 +1305,7 @@ def set_if_not_none(dictionary, key, value): # from the flex model field 'production-capacity' (a quantity) [-1] * 24 * 4, # from the power sensor attribute 'consumption_capacity' (a quantity) - [0.5] * 24 * 4, + [2] * 24 * 4, ), ( "Test battery", diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index e9083a6b5..3ba61a30f 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -373,41 +373,32 @@ def get_continuous_series_sensor_or_quantity( unit: ur.Quantity | str, query_window: tuple[datetime, datetime], resolution: timedelta, - default_value_attribute: str | None = None, - default_value: float | int | None = np.nan, + fallback_attribute: str | None = None, + max_value: float | int | None = np.nan, beliefs_before: datetime | None = None, - method: str = "replace", ) -> pd.Series: """ Retrieves a continuous time series data from a sensor or quantity within a specified window, filling - the missing values from an attribute (`default_value_attribute`) or default value (`default_value`). - - Methods to fill-in missing data: - - 'replace' missing values are filled with the default value. - - 'upper' clips missing values to the upper bound of the default value. - - 'lower' clips missing values to the lower bound of the default value. + the missing values from a given `fallback_attribute` and making sure no values exceed `max_value`. :param quantity_or_sensor: The quantity or sensor containing the data. :param actuator: The actuator from which relevant defaults are retrieved. :param unit: The desired unit of the data. :param query_window: The time window (start, end) to query the data. :param resolution: The resolution or time interval for the data. - :param default_value_attribute: Attribute for a default value if data is missing. - :param default_value: Default value if no attribute or data found. + :param fallback_attribute: Attribute serving as a fallback default in case of missing data. + :param max_value: Maximum value (also replacing NaN values). :param beliefs_before: Timestamp for prior beliefs or knowledge. - :param method: Method for handling missing data: 'replace', 'upper', 'lower', 'max', or 'min'. :returns: time series data with missing values handled based on the chosen method. - :raises: NotImplementedError: If an unsupported method is provided. """ _default_value = np.nan - if default_value_attribute is not None: + if fallback_attribute is not None: _default_value = get_quantity_from_attribute( entity=actuator, - attribute=default_value_attribute, + attribute=fallback_attribute, unit=unit, - default=default_value, ) time_series = get_series_from_quantity_or_sensor( @@ -418,15 +409,15 @@ def get_continuous_series_sensor_or_quantity( beliefs_before=beliefs_before, ) - if method == "replace": - time_series = time_series.fillna(_default_value) - elif method == "upper": - time_series = time_series.fillna(_default_value).clip(upper=_default_value) - elif method == "lower": - time_series = time_series.fillna(_default_value).clip(lower=_default_value) - else: - raise NotImplementedError( - "Method `{method}` not supported. Please, try one of the following: `replace`, `max`, `min` " - ) + # Use default as fallback + time_series = time_series.fillna(_default_value) + + # Apply upper limit + time_series = nanmin_of_series_and_value(time_series, max_value) return time_series + + +def nanmin_of_series_and_value(s: pd.Series, value: float) -> pd.Series: + """Perform a nanmin between a Series and a float.""" + return s.fillna(value).clip(upper=value) From c1b24dd55c6c152bd274860550aa9dd6fbd54c97 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 23 Nov 2023 20:49:40 +0100 Subject: [PATCH 2/2] fix: fill gaps in capacity sensor data using the max_value rather than with the fallback Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_solver.py | 10 ++++++---- flexmeasures/data/models/planning/utils.py | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index c50f06dbf..e1225b5e2 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -1269,8 +1269,9 @@ def set_if_not_none(dictionary, key, value): False, None, None, - # from the flex model field 'production-capacity' (a sensor) - [-0.2] * 4 * 4 + [-0.3] * 4 * 4 + [-8] * 16 * 4, + # from the flex model field 'production-capacity' (a sensor), + # and when absent, defaulting to the max value from the power sensor attribute capacity_in_mw + [-0.2] * 4 * 4 + [-0.3] * 4 * 4 + [-10] * 16 * 4, # from the power sensor attribute 'consumption_capacity' [0.5] * 24 * 4, ), @@ -1282,8 +1283,9 @@ def set_if_not_none(dictionary, key, value): None, # from the power sensor attribute 'consumption_capacity' [-8] * 24 * 4, - # from the flex model field 'consumption-capacity' (a sensor) - [0.25] * 4 * 4 + [0.15] * 4 * 4 + [0.5] * 16 * 4, + # from the flex model field 'consumption-capacity' (a sensor), + # and when absent, defaulting to the max value from the power sensor attribute capacity_in_mw + [0.25] * 4 * 4 + [0.15] * 4 * 4 + [10] * 16 * 4, ), ( "Test battery with dynamic power capacity", diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 3ba61a30f..64d92b56c 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -386,7 +386,7 @@ def get_continuous_series_sensor_or_quantity( :param unit: The desired unit of the data. :param query_window: The time window (start, end) to query the data. :param resolution: The resolution or time interval for the data. - :param fallback_attribute: Attribute serving as a fallback default in case of missing data. + :param fallback_attribute: Attribute serving as a fallback default in case no quantity or sensor is given. :param max_value: Maximum value (also replacing NaN values). :param beliefs_before: Timestamp for prior beliefs or knowledge. :returns: time series data with missing values handled based on the chosen method. @@ -410,7 +410,8 @@ def get_continuous_series_sensor_or_quantity( ) # Use default as fallback - time_series = time_series.fillna(_default_value) + if quantity_or_sensor is None: + time_series = time_series.fillna(_default_value) # Apply upper limit time_series = nanmin_of_series_and_value(time_series, max_value)