From 8280ee847201454bf1ac1a92d83209d140043619 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 10:32:05 +0200 Subject: [PATCH 01/19] style: sphinx docstrings Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 47 ++++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 7f8e6a5d0..752670bfa 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -41,10 +41,8 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: """Schedule a battery or Charge Point based directly on the latest beliefs regarding market prices within the specified time window. For the resulting consumption schedule, consumption is defined as positive values. - Args: - skip_validation: bool, default=False - whether to skip validation of constraints from data - + :param skip_validation: if True, skip validation of constraints specified in the data + :returns: the computed schedule """ if not self.config_deserialized: self.deserialize_config() @@ -440,22 +438,21 @@ def add_storage_constraints( soc_max: float, soc_min: float, ) -> pd.DataFrame: - """_summary_ - - Args: - storage_device_constraints (pd.DataFrame): _description_ - start (datetime): start of the schedule. - end (datetime): end of the schedule. - resolution (timedelta): timedelta used to resample the forecasts to the resolution of the schedule. - soc_at_start (float): state of charge at the start time. - soc_targets (List[Dict[str, datetime | float]] | pd.Series | None): list of (time : value) pairs - soc_maxima (List[Dict[str, datetime | float]] | pd.Series | None): _description_ - soc_minima (List[Dict[str, datetime | float]] | pd.Series | None): _description_ - soc_max (float): _description_ - soc_min (float): _description_ - - Returns: - pd.DataFrame: _description_ + """Collect all constraints for a given storage device in a DataFrame that the device_scheduler can interpret. + + :param storage_device_constraints: Empty frame without constraints (columns) for a storage device, + but already defining each time step (index). + :param start: Start of the schedule. + :param end: End of the schedule. + :param resolution: Timedelta used to resample the forecasts to the resolution of the schedule. + :param soc_at_start: State of charge at the start time. + :param soc_targets: Exact targets for the state of charge at each time. + :param soc_maxima: Maximum state of charge at each time. + :param soc_minima: Minimum state of charge at each time. + :param soc_max: Maximum state of charge at all times. + :param soc_min: Minimum state of charge at all times. + :returns: Constraints (columns) for a storage device, at each time step (index). + See device_scheduler for possible column names. """ if soc_targets is not None: @@ -529,10 +526,12 @@ def validate_storage_constraints( C.5) condition equals(t) - max(t-1) <= `derivative max`(t) C.6) `derivative min`(t) <= equals(t) - min(t-1) - Args: - storage_constraints: pd.DataFrame - dataframe containing the constraints of a storage device - + :param storage_constraints: dataframe containing the constraints of a storage device + :param soc_at_start: State of charge at the start time. + :param min_soc: Minimum state of charge at all times. + :param max_soc: Maximum state of charge at all times. + :param resolution: Constant duration between the start of each time step. + :returns: List of constraint violations, specifying their time, constraint and violation. """ constraint_violations = [] From af836466075e70fff336f898bd95efa1bec00b4f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 10:32:58 +0200 Subject: [PATCH 02/19] style: more detailed type annotation Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 752670bfa..d45605fe5 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -508,7 +508,7 @@ def validate_storage_constraints( min_soc: float, max_soc: float, resolution: timedelta, -) -> list: +) -> list[dict]: """Check that the storage constraints are fulfilled, e.g min <= equals <= max. A. Global validation From 8548777ea297bcc6e5bca9435239ed1b84dd4e88 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 10:35:09 +0200 Subject: [PATCH 03/19] style: consistency Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index d45605fe5..9d2182a03 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -41,8 +41,8 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: """Schedule a battery or Charge Point based directly on the latest beliefs regarding market prices within the specified time window. For the resulting consumption schedule, consumption is defined as positive values. - :param skip_validation: if True, skip validation of constraints specified in the data - :returns: the computed schedule + :param skip_validation: If True, skip validation of constraints specified in the data. + :returns: The computed schedule. """ if not self.config_deserialized: self.deserialize_config() From 8394901419823940bc8eb75cae3d21c73bde9426 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 10:36:03 +0200 Subject: [PATCH 04/19] fix: incomplete type annotation Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 9d2182a03..a0d06b91a 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -326,7 +326,7 @@ def get_min_max_targets( def get_min_max_soc_on_sensor( self, adjust_unit: bool = False, deserialized_names: bool = True - ) -> tuple[float | None]: + ) -> tuple[float | None, float | None]: soc_min_sensor = self.sensor.get_attribute("min_soc_in_mwh", None) soc_max_sensor = self.sensor.get_attribute("max_soc_in_mwh", None) soc_unit_label = "soc_unit" if deserialized_names else "soc-unit" From 59ec68a55e6e00fa7adf4b8080d53da0b6ff9e28 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 10:39:20 +0200 Subject: [PATCH 05/19] fix: validation A.1) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 8 ++++---- flexmeasures/data/models/planning/tests/test_solver.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index a0d06b91a..fe8af018b 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -512,7 +512,7 @@ def validate_storage_constraints( """Check that the storage constraints are fulfilled, e.g min <= equals <= max. A. Global validation - A.1) min <= min_soc + A.1) min >= min_soc A.2) max <= max_soc B. Validation in the same time frame B.1) min <= max @@ -541,7 +541,7 @@ def validate_storage_constraints( ######################## min_soc = (min_soc - soc_at_start) * timedelta(hours=1) / resolution - # 1) min <= min_soc + # 1) min >= min_soc mask = ~(storage_constraints["min"] >= min_soc) time_condition_fails = storage_constraints.index[mask] @@ -551,8 +551,8 @@ def validate_storage_constraints( constraint_violations.append( dict( dt=dt.to_pydatetime(), - condition="min <= min_soc", - violation=f"min [{value_min}] <= min_soc [{min_soc}]", + condition="min >= min_soc", + violation=f"min [{value_min}] >= min_soc [{min_soc}]", ) ) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 8b436acae..08a1cb7bd 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -607,7 +607,7 @@ def test_add_storage_constraints( "value_min1, value_equals1, value_max1, value_min2, value_equals2, value_max2, expected_constraint_type_violations", [ (1, np.nan, 9, 2, np.nan, 20, ["max <= max_soc"]), - (-1, np.nan, 9, 1, np.nan, 9, ["min <= min_soc"]), + (-1, np.nan, 9, 1, np.nan, 9, ["min >= min_soc"]), (1, 10, 9, 1, np.nan, 9, ["equals <= max"]), (1, 0, 9, 1, np.nan, 9, ["min <= equals"]), ( From cc532db2867661d1cca882cc7e49aba5b9a81b0c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 10:53:04 +0200 Subject: [PATCH 06/19] fix: Validation B.2) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 4 ++-- flexmeasures/data/models/planning/tests/test_solver.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index fe8af018b..d5c747ba2 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -593,7 +593,7 @@ def validate_storage_constraints( ) ) - # 2) min <= equals + # 2) equals >= min mask = ~(storage_constraints["equals"] >= storage_constraints["min"]) mask = mask & ~storage_constraints["equals"].isna() time_condition_fails = storage_constraints.index[mask] @@ -605,7 +605,7 @@ def validate_storage_constraints( constraint_violations.append( dict( dt=dt.to_pydatetime(), - condition="min <= equals", + condition="equals >= min", violation=f"equal [{value_equals}] >= min [{value_min}]", ) ) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 08a1cb7bd..5fa9dade1 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -609,7 +609,7 @@ def test_add_storage_constraints( (1, np.nan, 9, 2, np.nan, 20, ["max <= max_soc"]), (-1, np.nan, 9, 1, np.nan, 9, ["min >= min_soc"]), (1, 10, 9, 1, np.nan, 9, ["equals <= max"]), - (1, 0, 9, 1, np.nan, 9, ["min <= equals"]), + (1, 0, 9, 1, np.nan, 9, ["equals >= min"]), ( 1, np.nan, @@ -619,7 +619,7 @@ def test_add_storage_constraints( 1, ["min <= max"], ), - (9, 5, 1, 1, np.nan, 9, ["min <= equals", "equals <= max", "min <= max"]), + (9, 5, 1, 1, np.nan, 9, ["equals >= min", "equals <= max", "min <= max"]), (1, np.nan, 9, 1, np.nan, 9, []), # same interval, should not fail (1, np.nan, 9, 3, np.nan, 7, []), # should not fail, containing interval (1, np.nan, 3, 3, np.nan, 5, []), # difference = 0 < 1, should not fail From 8d7d1d0bc8ca2cd4c0f09cc301e9d561536baba4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 10:57:08 +0200 Subject: [PATCH 07/19] fix: Validation C.1) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index d5c747ba2..629e0a394 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -655,7 +655,7 @@ def validate_storage_constraints( dict( dt=dt.to_pydatetime(), condition="equals(t) - equals(t-1) <= `derivative max`(t)", - violation=f"equals(t) [{value_equals}] - equals(t-1) [{value_equals_previous}] <= max [{value_derivative_max}]", + violation=f"equals(t) [{value_equals}] - equals(t-1) [{value_equals_previous}] <= `derivative max`(t) [{value_derivative_max}]", ) ) From d5dc5810f211daeef31d1e217a98acddf42c63e8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 10:59:32 +0200 Subject: [PATCH 08/19] fix: Validation C.3) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 629e0a394..88c095bf5 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -714,7 +714,7 @@ def validate_storage_constraints( dict( dt=dt.to_pydatetime(), condition="min(t) - max(t-1) <= `derivative max`(t)", - violation=f"min(t) [{value_min}] <= max(t-1) [{value_max_previous}] + `derivative max` [{value_derivative_max}]", + violation=f"min(t) [{value_min}] - max(t-1) [{value_max_previous}] <= `derivative max`(t) [{value_derivative_max}]", ) ) From 97da1c97cc5c0de141c4a814c77383019fa250b1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 11:00:41 +0200 Subject: [PATCH 09/19] fix: Validation C.4) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 88c095bf5..221c3283e 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -732,7 +732,7 @@ def validate_storage_constraints( dict( dt=dt.to_pydatetime(), condition="max(t) - min(t-1) >= `derivative min`", - violation=f"max(t) [{value_max}] - min(t-1) [{value_min_previous}]<= `derivative min` [{value_derivative_min}]", + violation=f"max(t) [{value_max}] - min(t-1) [{value_min_previous}] >= `derivative min`(t) [{value_derivative_min}]", ) ) From fa49442cfb1a6b5032058311a573b9d78d063c0b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 11:05:17 +0200 Subject: [PATCH 10/19] refactor: move statements to where they matter Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 221c3283e..7a704b7c5 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -694,13 +694,10 @@ def validate_storage_constraints( min_extended[storage_constraints.index[0] - resolution] = min_soc min_extended = min_extended.sort_index() + # 3) min(t) - max(t-1) <= `derivative max`(t) delta_min_max = min_extended - max_extended.shift(1) delta_min_max = delta_min_max[1:] - delta_max_min = max_extended - min_extended.shift(1) - delta_max_min = delta_max_min[1:] - - # 3) min(t) - max(t-1) <= `derivative max`(t) condition3 = delta_min_max <= storage_constraints["derivative max"] * factor_w_wh mask = ~condition3 time_condition_fails = storage_constraints.index[mask] @@ -719,6 +716,9 @@ def validate_storage_constraints( ) # 4) max(t) - min(t-1) >= `derivative min`(t) + delta_max_min = max_extended - min_extended.shift(1) + delta_max_min = delta_max_min[1:] + condition4 = delta_max_min >= storage_constraints["derivative min"] * factor_w_wh mask = ~condition4 time_condition_fails = storage_constraints.index[mask] From 47b067b3950531a70e8b72bf36ca72b11ba0720f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 11:21:33 +0200 Subject: [PATCH 11/19] style: consistency (also: https://books.google.com/ngrams/graph?content=infeasible%2Cunfeasible&year_start=1800&year_end=2019&corpus=en-2019&smoothing=0&case_insensitive=true ) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 7a704b7c5..b7e4491aa 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -179,7 +179,7 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: if len(constraint_violations) > 0: # TODO: include hints from constraint_violations into the error message - raise ValueError("The input data yields an unfeasible problem.") + raise ValueError("The input data yields an infeasible problem.") # Set up EMS constraints columns = ["derivative max", "derivative min"] From 63ed923bc3422a93bf80d6082653c208e977d8bd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 11:24:33 +0200 Subject: [PATCH 12/19] style: Validation B.2) Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 8 ++++---- flexmeasures/data/models/planning/tests/test_solver.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index b7e4491aa..7b7286efa 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -593,8 +593,8 @@ def validate_storage_constraints( ) ) - # 2) equals >= min - mask = ~(storage_constraints["equals"] >= storage_constraints["min"]) + # 2) min <= equals + mask = ~(storage_constraints["min"] <= storage_constraints["equals"]) mask = mask & ~storage_constraints["equals"].isna() time_condition_fails = storage_constraints.index[mask] @@ -605,8 +605,8 @@ def validate_storage_constraints( constraint_violations.append( dict( dt=dt.to_pydatetime(), - condition="equals >= min", - violation=f"equal [{value_equals}] >= min [{value_min}]", + condition="min <= equals", + violation=f"min [{value_min}] <= equal [{value_equals}]", ) ) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 5fa9dade1..08a1cb7bd 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -609,7 +609,7 @@ def test_add_storage_constraints( (1, np.nan, 9, 2, np.nan, 20, ["max <= max_soc"]), (-1, np.nan, 9, 1, np.nan, 9, ["min >= min_soc"]), (1, 10, 9, 1, np.nan, 9, ["equals <= max"]), - (1, 0, 9, 1, np.nan, 9, ["equals >= min"]), + (1, 0, 9, 1, np.nan, 9, ["min <= equals"]), ( 1, np.nan, @@ -619,7 +619,7 @@ def test_add_storage_constraints( 1, ["min <= max"], ), - (9, 5, 1, 1, np.nan, 9, ["equals >= min", "equals <= max", "min <= max"]), + (9, 5, 1, 1, np.nan, 9, ["min <= equals", "equals <= max", "min <= max"]), (1, np.nan, 9, 1, np.nan, 9, []), # same interval, should not fail (1, np.nan, 9, 3, np.nan, 7, []), # should not fail, containing interval (1, np.nan, 3, 3, np.nan, 5, []), # difference = 0 < 1, should not fail From f0072043ba2a0cf6f7dbf500347a2b953fa8f677 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 May 2023 12:43:58 +0200 Subject: [PATCH 13/19] refactor: Constraint validation for cases A and B Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 215 ++++++++---------- .../data/models/planning/tests/test_solver.py | 2 +- 2 files changed, 101 insertions(+), 116 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 7b7286efa..7c194b336 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -167,10 +167,10 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: ) device_constraints[0]["derivative up efficiency"] = roundtrip_efficiency**0.5 - # check that storage constraints are fullfiled + # check that storage constraints are fulfilled if not skip_validation: constraint_violations = validate_storage_constraints( - storage_constraints=device_constraints[0], + constraints=device_constraints[0], soc_at_start=soc_at_start, min_soc=soc_min, max_soc=soc_max, @@ -503,7 +503,7 @@ def add_storage_constraints( def validate_storage_constraints( - storage_constraints: pd.DataFrame, + constraints: pd.DataFrame, soc_at_start: float, min_soc: float, max_soc: float, @@ -526,7 +526,7 @@ def validate_storage_constraints( C.5) condition equals(t) - max(t-1) <= `derivative max`(t) C.6) `derivative min`(t) <= equals(t) - min(t-1) - :param storage_constraints: dataframe containing the constraints of a storage device + :param constraints: dataframe containing the constraints of a storage device :param soc_at_start: State of charge at the start time. :param min_soc: Minimum state of charge at all times. :param max_soc: Maximum state of charge at all times. @@ -539,94 +539,39 @@ def validate_storage_constraints( ######################## # A. Global validation # ######################## - min_soc = (min_soc - soc_at_start) * timedelta(hours=1) / resolution # 1) min >= min_soc - mask = ~(storage_constraints["min"] >= min_soc) - time_condition_fails = storage_constraints.index[mask] - - for dt in time_condition_fails: - value_min = storage_constraints.loc[dt, "min"] - - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="min >= min_soc", - violation=f"min [{value_min}] >= min_soc [{min_soc}]", - ) - ) + min_soc = (min_soc - soc_at_start) * timedelta(hours=1) / resolution + constraint_violations += validate_constraint( + constraints, + "min", + ">=", + "min_soc", + right_value=min_soc, + ) # 2) max <= max_soc max_soc = (max_soc - soc_at_start) * timedelta(hours=1) / resolution - - mask = ~(storage_constraints["max"] <= max_soc) - time_condition_fails = storage_constraints.index[mask] - - for dt in time_condition_fails: - value_max = storage_constraints.loc[dt, "max"] - - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="max <= max_soc", - violation=f"max [{value_max}] <= max_soc [{max_soc}]", - ) - ) + constraint_violations += validate_constraint( + constraints, + "max", + "<=", + "max_soc", + right_value=max_soc, + ) ######################################## # B. Validation in the same time frame # ######################################## # 1) min <= max - mask = ~(storage_constraints["min"] <= storage_constraints["max"]) - time_condition_fails = storage_constraints.index[mask] - - for dt in time_condition_fails: - value_min = storage_constraints.loc[dt, "min"] - value_max = storage_constraints.loc[dt, "max"] - - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="min <= max", - violation=f"min [{value_min}] <= max [{value_max}]", - ) - ) + constraint_violations += validate_constraint(constraints, "min", "<=", "max") # 2) min <= equals - mask = ~(storage_constraints["min"] <= storage_constraints["equals"]) - mask = mask & ~storage_constraints["equals"].isna() - time_condition_fails = storage_constraints.index[mask] - - for dt in time_condition_fails: - value_equals = storage_constraints.loc[dt, "equals"] - value_min = storage_constraints.loc[dt, "min"] - - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="min <= equals", - violation=f"min [{value_min}] <= equal [{value_equals}]", - ) - ) + constraint_violations += validate_constraint(constraints, "min", "<=", "equals") # 3) equals <= max - mask = ~(storage_constraints["equals"] <= storage_constraints["max"]) - mask = mask & ~storage_constraints["equals"].isna() - - time_condition_fails = storage_constraints.index[mask] - - for dt in time_condition_fails: - value_equals = storage_constraints.loc[dt, "equals"] - value_max = storage_constraints.loc[dt, "max"] - - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="equals <= max", - violation=f"equals [{value_equals}] <= max [{value_max}]", - ) - ) + constraint_violations += validate_constraint(constraints, "equals", "<=", "max") ########################################## # C. Validation in different time frames # @@ -635,21 +580,21 @@ def validate_storage_constraints( factor_w_wh = resolution / timedelta(hours=1) # 1) equals(t) - equals(t-1) <= `derivative max`(t) - equals_extended = storage_constraints["equals"].copy() - equals_extended[storage_constraints.index[0] - resolution] = soc_at_start + equals_extended = constraints["equals"].copy() + equals_extended[constraints.index[0] - resolution] = soc_at_start equals_extended = equals_extended.sort_index() diff_equals = equals_extended.diff()[1:] mask = ( - ~(diff_equals <= storage_constraints["derivative max"] * factor_w_wh) + ~(diff_equals <= constraints["derivative max"] * factor_w_wh) & ~diff_equals.isna() ) - time_condition_fails = storage_constraints.index[mask] + time_condition_fails = constraints.index[mask] for dt in time_condition_fails: - value_equals = storage_constraints.loc[dt, "equals"] - value_equals_previous = storage_constraints.loc[dt - resolution, "equals"] - value_derivative_max = storage_constraints.loc[dt, "derivative max"] + value_equals = constraints.loc[dt, "equals"] + value_equals_previous = constraints.loc[dt - resolution, "equals"] + value_derivative_max = constraints.loc[dt, "derivative max"] constraint_violations.append( dict( @@ -660,21 +605,21 @@ def validate_storage_constraints( ) # 2) `derivative min`(t) <= equals(t) - equals(t-1) - equals_extended = storage_constraints["equals"].copy() - equals_extended[storage_constraints.index[0] - resolution] = soc_at_start + equals_extended = constraints["equals"].copy() + equals_extended[constraints.index[0] - resolution] = soc_at_start equals_extended = equals_extended.sort_index() diff_equals = equals_extended.diff()[1:] mask = ( - ~((storage_constraints["derivative min"] * factor_w_wh) <= diff_equals) + ~((constraints["derivative min"] * factor_w_wh) <= diff_equals) & ~diff_equals.isna() ) - time_condition_fails = storage_constraints.index[mask] + time_condition_fails = constraints.index[mask] for dt in time_condition_fails: - value_equals = storage_constraints.loc[dt, "equals"] - value_equals_previous = storage_constraints.loc[dt - resolution, "equals"] - value_derivative_min = storage_constraints.loc[dt, "derivative min"] + value_equals = constraints.loc[dt, "equals"] + value_equals_previous = constraints.loc[dt - resolution, "equals"] + value_derivative_min = constraints.loc[dt, "derivative min"] constraint_violations.append( dict( @@ -685,27 +630,27 @@ def validate_storage_constraints( ) # extend max - max_extended = storage_constraints["max"].copy() - max_extended[storage_constraints.index[0] - resolution] = max_soc + max_extended = constraints["max"].copy() + max_extended[constraints.index[0] - resolution] = max_soc max_extended = max_extended.sort_index() # extend min - min_extended = storage_constraints["min"].copy() - min_extended[storage_constraints.index[0] - resolution] = min_soc + min_extended = constraints["min"].copy() + min_extended[constraints.index[0] - resolution] = min_soc min_extended = min_extended.sort_index() # 3) min(t) - max(t-1) <= `derivative max`(t) delta_min_max = min_extended - max_extended.shift(1) delta_min_max = delta_min_max[1:] - condition3 = delta_min_max <= storage_constraints["derivative max"] * factor_w_wh + condition3 = delta_min_max <= constraints["derivative max"] * factor_w_wh mask = ~condition3 - time_condition_fails = storage_constraints.index[mask] + time_condition_fails = constraints.index[mask] for dt in time_condition_fails: - value_min = storage_constraints.loc[dt, "min"] + value_min = constraints.loc[dt, "min"] value_max_previous = max_extended.loc[dt - resolution] - value_derivative_max = storage_constraints.loc[dt, "derivative max"] + value_derivative_max = constraints.loc[dt, "derivative max"] constraint_violations.append( dict( @@ -719,14 +664,14 @@ def validate_storage_constraints( delta_max_min = max_extended - min_extended.shift(1) delta_max_min = delta_max_min[1:] - condition4 = delta_max_min >= storage_constraints["derivative min"] * factor_w_wh + condition4 = delta_max_min >= constraints["derivative min"] * factor_w_wh mask = ~condition4 - time_condition_fails = storage_constraints.index[mask] + time_condition_fails = constraints.index[mask] for dt in time_condition_fails: - value_max = storage_constraints.loc[dt, "max"] + value_max = constraints.loc[dt, "max"] value_min_previous = min_extended.loc[dt - resolution] - value_derivative_min = storage_constraints.loc[dt, "derivative min"] + value_derivative_min = constraints.loc[dt, "derivative min"] constraint_violations.append( dict( @@ -737,17 +682,17 @@ def validate_storage_constraints( ) # 5) equals(t) - max(t-1) <= `derivative max`(t) - delta_equals_max = storage_constraints["equals"] - max_extended.shift(1) + delta_equals_max = constraints["equals"] - max_extended.shift(1) delta_equals_max = delta_equals_max[1:] - condition5 = delta_equals_max <= storage_constraints["derivative max"] * factor_w_wh - mask = ~condition5 & ~storage_constraints["equals"].isna() - time_condition_fails = storage_constraints.index[mask] + condition5 = delta_equals_max <= constraints["derivative max"] * factor_w_wh + mask = ~condition5 & ~constraints["equals"].isna() + time_condition_fails = constraints.index[mask] for dt in time_condition_fails: - value_equals = storage_constraints.loc[dt, "equals"] + value_equals = constraints.loc[dt, "equals"] value_max_previous = max_extended.loc[dt - resolution] - value_derivative_max = storage_constraints.loc[dt, "derivative max"] + value_derivative_max = constraints.loc[dt, "derivative max"] constraint_violations.append( dict( @@ -758,17 +703,17 @@ def validate_storage_constraints( ) # 6) `derivative min`(t) <= equals(t) - min(t-1) - delta_equals_min = storage_constraints["equals"] - min_extended.shift(1) + delta_equals_min = constraints["equals"] - min_extended.shift(1) delta_equals_min = delta_equals_min[1:] - condition5 = delta_equals_min >= storage_constraints["derivative min"] * factor_w_wh - mask = ~condition5 & ~storage_constraints["equals"].isna() - time_condition_fails = storage_constraints.index[mask] + condition5 = delta_equals_min >= constraints["derivative min"] * factor_w_wh + mask = ~condition5 & ~constraints["equals"].isna() + time_condition_fails = constraints.index[mask] for dt in time_condition_fails: - value_equals = storage_constraints.loc[dt, "equals"] + value_equals = constraints.loc[dt, "equals"] value_min_previous = min_extended.loc[dt - resolution] - value_derivative_min = storage_constraints.loc[dt, "derivative min"] + value_derivative_min = constraints.loc[dt, "derivative min"] constraint_violations.append( dict( @@ -781,6 +726,46 @@ def validate_storage_constraints( return constraint_violations +def validate_constraint( + constraints, + left_constraint_name, + inequality, + right_constraint_name, + left_value: float | None = None, + right_value: float | None = None, +) -> list[dict]: + """Validate the feasibility of a given set of constraints. + + :returns: List of constraint violations, specifying their time, constraint and violation. + """ + mask = True + if left_value is None: + left_value = constraints[left_constraint_name] + mask = mask & ~constraints[left_constraint_name].isna() + if right_value is None: + right_value = constraints[right_constraint_name] + mask = mask & ~constraints[right_constraint_name].isna() + if inequality == "<=": + mask = mask & ~(left_value <= right_value) + elif inequality == ">=": + mask = mask & ~(left_value >= right_value) + else: + raise NotImplementedError(f"Inequality '{inequality}' not supported.") + time_condition_fails = constraints.index[mask] + constraint_violations = [] + for dt in time_condition_fails: + lv = left_value[dt] if isinstance(left_value, pd.Series) else left_value + rv = right_value[dt] if isinstance(right_value, pd.Series) else right_value + constraint_violations.append( + dict( + dt=dt.to_pydatetime(), + condition=f"{left_constraint_name} {inequality} {right_constraint_name}", + violation=f"{left_constraint_name} [{lv}] {inequality} {right_constraint_name} [{rv}]", + ) + ) + return constraint_violations + + ##################### # TO BE DEPRECATED # #################### diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 08a1cb7bd..1581175fb 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -717,7 +717,7 @@ def test_validate_constraints( ] = value_equals2 constraint_violations = validate_storage_constraints( - storage_constraints=storage_device_constraints, + constraints=storage_device_constraints, soc_at_start=0.0, min_soc=0, max_soc=10, From 0f39b8c1f70db81594cc6bcfd003760a57b73895 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 25 May 2023 13:04:29 +0200 Subject: [PATCH 14/19] style: fixing typos Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/tests/test_solver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 1581175fb..9a5c286f7 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -439,7 +439,7 @@ def test_soc_bounds_timeseries(add_battery_assets): battery = Sensor.query.filter(Sensor.name == "Test battery").one_or_none() assert battery.get_attribute("market_id") == epex_da.id - # time paramaters + # time parameters tz = pytz.timezone("Europe/Amsterdam") start = tz.localize(datetime(2015, 1, 2)) end = tz.localize(datetime(2015, 1, 3)) @@ -623,7 +623,7 @@ def test_add_storage_constraints( (1, np.nan, 9, 1, np.nan, 9, []), # same interval, should not fail (1, np.nan, 9, 3, np.nan, 7, []), # should not fail, containing interval (1, np.nan, 3, 3, np.nan, 5, []), # difference = 0 < 1, should not fail - (1, np.nan, 3, 4, np.nan, 5, []), # difference == max, should not fails + (1, np.nan, 3, 4, np.nan, 5, []), # difference == max, should not fail ( 1, np.nan, From 4c8b91246e1d1b0bbdc94358509add3125dcc62e Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 25 May 2023 13:32:40 +0200 Subject: [PATCH 15/19] fix: add comments clarifying why we need to sort the `{equals, max, min}_extended` series Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 7c194b336..0f668b16a 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -579,12 +579,17 @@ def validate_storage_constraints( factor_w_wh = resolution / timedelta(hours=1) - # 1) equals(t) - equals(t-1) <= `derivative max`(t) + # compute diff_equals(t) = equals(t) - equals(t-1) equals_extended = constraints["equals"].copy() + # insert `soc_at_start` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series equals_extended[constraints.index[0] - resolution] = soc_at_start + # sort index to keep the time ordering equals_extended = equals_extended.sort_index() + diff_equals = equals_extended.diff()[1:] + # 1) equals(t) - equals(t-1) <= `derivative max`(t) + mask = ( ~(diff_equals <= constraints["derivative max"] * factor_w_wh) & ~diff_equals.isna() @@ -605,10 +610,6 @@ def validate_storage_constraints( ) # 2) `derivative min`(t) <= equals(t) - equals(t-1) - equals_extended = constraints["equals"].copy() - equals_extended[constraints.index[0] - resolution] = soc_at_start - equals_extended = equals_extended.sort_index() - diff_equals = equals_extended.diff()[1:] mask = ( ~((constraints["derivative min"] * factor_w_wh) <= diff_equals) @@ -631,12 +632,16 @@ def validate_storage_constraints( # extend max max_extended = constraints["max"].copy() + # insert `max_soc` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series max_extended[constraints.index[0] - resolution] = max_soc + # sort index to keep the time ordering max_extended = max_extended.sort_index() # extend min min_extended = constraints["min"].copy() + # insert `max_soc` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series min_extended[constraints.index[0] - resolution] = min_soc + # sort index to keep the time ordering min_extended = min_extended.sort_index() # 3) min(t) - max(t-1) <= `derivative max`(t) From 8c1d7635474b7b9699e581c427b90481c243b17b Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 25 May 2023 14:01:25 +0200 Subject: [PATCH 16/19] refactor: create storage_device_constraints inside of add_storage_constraints Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 42 ++++++++++--------- .../data/models/planning/tests/test_solver.py | 31 +++----------- 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 0f668b16a..cf5534a7f 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -28,6 +28,17 @@ class StorageScheduler(Scheduler): __version__ = "1" __author__ = "Seita" + COLUMNS = [ + "equals", + "max", + "min", + "derivative equals", + "derivative max", + "derivative min", + "derivative down efficiency", + "derivative up efficiency", + ] + def compute_schedule(self) -> pd.Series | None: """Schedule a battery or Charge Point based directly on the latest beliefs regarding market prices within the specified time window. For the resulting consumption schedule, consumption is defined as positive values. @@ -52,12 +63,13 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: resolution = self.resolution belief_time = self.belief_time sensor = self.sensor + soc_at_start = self.flex_model.get("soc_at_start") soc_targets = self.flex_model.get("soc_targets") soc_min = self.flex_model.get("soc_min") soc_max = self.flex_model.get("soc_max") - soc_maxima = self.flex_model.get("soc_maxima") soc_minima = self.flex_model.get("soc_minima") + soc_maxima = self.flex_model.get("soc_maxima") roundtrip_efficiency = self.flex_model.get("roundtrip_efficiency") prefer_charging_sooner = self.flex_model.get("prefer_charging_sooner", True) @@ -113,18 +125,8 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: ] # Set up device constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n). - columns = [ - "equals", - "max", - "min", - "derivative equals", - "derivative max", - "derivative min", - "derivative down efficiency", - "derivative up efficiency", - ] device_constraints = [ - initialize_df(columns, start, end, resolution) + initialize_df(StorageScheduler.COLUMNS, start, end, resolution) for i in range(1 + len(inflexible_device_sensors)) ] for i, inflexible_sensor in enumerate(inflexible_device_sensors): @@ -136,7 +138,6 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: ) device_constraints[0] = add_storage_constraints( - device_constraints[0], start, end, resolution, @@ -182,8 +183,9 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: raise ValueError("The input data yields an infeasible problem.") # Set up EMS constraints - columns = ["derivative max", "derivative min"] - ems_constraints = initialize_df(columns, start, end, resolution) + ems_constraints = initialize_df( + StorageScheduler.COLUMNS, start, end, resolution + ) ems_capacity = sensor.generic_asset.get_attribute("capacity_in_mw") if ems_capacity is not None: ems_constraints["derivative min"] = ems_capacity * -1 @@ -427,7 +429,6 @@ def build_device_soc_values( def add_storage_constraints( - storage_device_constraints: pd.DataFrame, start: datetime, end: datetime, resolution: timedelta, @@ -440,8 +441,6 @@ def add_storage_constraints( ) -> pd.DataFrame: """Collect all constraints for a given storage device in a DataFrame that the device_scheduler can interpret. - :param storage_device_constraints: Empty frame without constraints (columns) for a storage device, - but already defining each time step (index). :param start: Start of the schedule. :param end: End of the schedule. :param resolution: Timedelta used to resample the forecasts to the resolution of the schedule. @@ -451,10 +450,15 @@ def add_storage_constraints( :param soc_minima: Minimum state of charge at each time. :param soc_max: Maximum state of charge at all times. :param soc_min: Minimum state of charge at all times. - :returns: Constraints (columns) for a storage device, at each time step (index). + :returns: Constraints (StorageScheduler.COLUMNS) for a storage device, at each time step (index). See device_scheduler for possible column names. """ + # create empty storage device constraints dataframe + storage_device_constraints = initialize_df( + StorageScheduler.COLUMNS, start, end, resolution + ) + if soc_targets is not None: # make an equality series with the SOC targets set in the flex model # storage_device_constraints refers to the flexible device we are scheduling diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 9a5c286f7..d40441535 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -523,22 +523,15 @@ def compute_schedule(flex_model): @pytest.mark.parametrize( - "value_soc_min, value_soc_minima, value_soc_target, value_soc_maxima, value_soc_max, min, equals, max", + "value_soc_min, value_soc_minima, value_soc_target, value_soc_maxima, value_soc_max", [ - (-1, -0.5, 0, 0.5, 1, -0.5, 0, 0.5), - (-1, -2, 0, 0.5, 1, -1, 0, 0.5), - (-1, -0.5, 0.5, 0.5, 1, -0.5, 0.5, 0.5), + (-1, -0.5, 0, 0.5, 1.0), + (-1, -2, 0, 0.5, 1.0), + (-1, -0.5, 0.5, 0.5, 1.0), ], ) def test_add_storage_constraints( - value_soc_min, - value_soc_minima, - value_soc_target, - value_soc_maxima, - value_soc_max, - min, - equals, - max, + value_soc_min, value_soc_minima, value_soc_target, value_soc_maxima, value_soc_max ): """Check that the storage constraints are generated properly""" @@ -550,17 +543,6 @@ def test_add_storage_constraints( soc_at_start = 0.0 - columns = [ - "equals", - "max", - "min", - "derivative equals", - "derivative max", - "derivative min", - "derivative down efficiency", - "derivative up efficiency", - ] - test_date = start + timedelta(hours=1) soc_targets = initialize_series(np.nan, start, end, resolution) @@ -575,10 +557,7 @@ def test_add_storage_constraints( soc_max = value_soc_max soc_min = value_soc_min - storage_device_constraints = initialize_df(columns, start, end, resolution) - storage_device_constraints = add_storage_constraints( - storage_device_constraints, start, end, resolution, From 649f1d7c3210bc4ba6830a5126b6096f6ad0bbb3 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 25 May 2023 14:12:47 +0200 Subject: [PATCH 17/19] reactor: max_soc -> soc_max Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 22 +++++++++---------- .../data/models/planning/tests/test_solver.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index cf5534a7f..04cb023a3 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -174,7 +174,7 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: constraints=device_constraints[0], soc_at_start=soc_at_start, min_soc=soc_min, - max_soc=soc_max, + soc_max=soc_max, resolution=resolution, ) @@ -510,14 +510,14 @@ def validate_storage_constraints( constraints: pd.DataFrame, soc_at_start: float, min_soc: float, - max_soc: float, + soc_max: float, resolution: timedelta, ) -> list[dict]: """Check that the storage constraints are fulfilled, e.g min <= equals <= max. A. Global validation A.1) min >= min_soc - A.2) max <= max_soc + A.2) max <= soc_max B. Validation in the same time frame B.1) min <= max B.2) min <= equals @@ -533,7 +533,7 @@ def validate_storage_constraints( :param constraints: dataframe containing the constraints of a storage device :param soc_at_start: State of charge at the start time. :param min_soc: Minimum state of charge at all times. - :param max_soc: Maximum state of charge at all times. + :param soc_max: Maximum state of charge at all times. :param resolution: Constant duration between the start of each time step. :returns: List of constraint violations, specifying their time, constraint and violation. """ @@ -554,14 +554,14 @@ def validate_storage_constraints( right_value=min_soc, ) - # 2) max <= max_soc - max_soc = (max_soc - soc_at_start) * timedelta(hours=1) / resolution + # 2) max <= soc_max + soc_max = (soc_max - soc_at_start) * timedelta(hours=1) / resolution constraint_violations += validate_constraint( constraints, "max", "<=", - "max_soc", - right_value=max_soc, + "soc_max", + right_value=soc_max, ) ######################################## @@ -636,14 +636,14 @@ def validate_storage_constraints( # extend max max_extended = constraints["max"].copy() - # insert `max_soc` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series - max_extended[constraints.index[0] - resolution] = max_soc + # insert `soc_max` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series + max_extended[constraints.index[0] - resolution] = soc_max # sort index to keep the time ordering max_extended = max_extended.sort_index() # extend min min_extended = constraints["min"].copy() - # insert `max_soc` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series + # insert `soc_max` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series min_extended[constraints.index[0] - resolution] = min_soc # sort index to keep the time ordering min_extended = min_extended.sort_index() diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index d40441535..dc0b646a4 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -585,7 +585,7 @@ def test_add_storage_constraints( @pytest.mark.parametrize( "value_min1, value_equals1, value_max1, value_min2, value_equals2, value_max2, expected_constraint_type_violations", [ - (1, np.nan, 9, 2, np.nan, 20, ["max <= max_soc"]), + (1, np.nan, 9, 2, np.nan, 20, ["max <= soc_max"]), (-1, np.nan, 9, 1, np.nan, 9, ["min >= min_soc"]), (1, 10, 9, 1, np.nan, 9, ["equals <= max"]), (1, 0, 9, 1, np.nan, 9, ["min <= equals"]), @@ -699,7 +699,7 @@ def test_validate_constraints( constraints=storage_device_constraints, soc_at_start=0.0, min_soc=0, - max_soc=10, + soc_max=10, resolution=resolution, ) From 674d2599843509aebeac1c284853158e3a88467e Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 25 May 2023 14:27:36 +0200 Subject: [PATCH 18/19] reactor: min_soc -> soc_min Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 18 +++++++++--------- .../data/models/planning/tests/test_solver.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 04cb023a3..d3a6710a7 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -173,7 +173,7 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: constraint_violations = validate_storage_constraints( constraints=device_constraints[0], soc_at_start=soc_at_start, - min_soc=soc_min, + soc_min=soc_min, soc_max=soc_max, resolution=resolution, ) @@ -509,14 +509,14 @@ def add_storage_constraints( def validate_storage_constraints( constraints: pd.DataFrame, soc_at_start: float, - min_soc: float, + soc_min: float, soc_max: float, resolution: timedelta, ) -> list[dict]: """Check that the storage constraints are fulfilled, e.g min <= equals <= max. A. Global validation - A.1) min >= min_soc + A.1) min >= soc_min A.2) max <= soc_max B. Validation in the same time frame B.1) min <= max @@ -532,7 +532,7 @@ def validate_storage_constraints( :param constraints: dataframe containing the constraints of a storage device :param soc_at_start: State of charge at the start time. - :param min_soc: Minimum state of charge at all times. + :param soc_min: Minimum state of charge at all times. :param soc_max: Maximum state of charge at all times. :param resolution: Constant duration between the start of each time step. :returns: List of constraint violations, specifying their time, constraint and violation. @@ -544,14 +544,14 @@ def validate_storage_constraints( # A. Global validation # ######################## - # 1) min >= min_soc - min_soc = (min_soc - soc_at_start) * timedelta(hours=1) / resolution + # 1) min >= soc_min + soc_min = (soc_min - soc_at_start) * timedelta(hours=1) / resolution constraint_violations += validate_constraint( constraints, "min", ">=", - "min_soc", - right_value=min_soc, + "soc_min", + right_value=soc_min, ) # 2) max <= soc_max @@ -644,7 +644,7 @@ def validate_storage_constraints( # extend min min_extended = constraints["min"].copy() # insert `soc_max` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series - min_extended[constraints.index[0] - resolution] = min_soc + min_extended[constraints.index[0] - resolution] = soc_min # sort index to keep the time ordering min_extended = min_extended.sort_index() diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index dc0b646a4..b644743a0 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -586,7 +586,7 @@ def test_add_storage_constraints( "value_min1, value_equals1, value_max1, value_min2, value_equals2, value_max2, expected_constraint_type_violations", [ (1, np.nan, 9, 2, np.nan, 20, ["max <= soc_max"]), - (-1, np.nan, 9, 1, np.nan, 9, ["min >= min_soc"]), + (-1, np.nan, 9, 1, np.nan, 9, ["min >= soc_min"]), (1, 10, 9, 1, np.nan, 9, ["equals <= max"]), (1, 0, 9, 1, np.nan, 9, ["min <= equals"]), ( @@ -698,7 +698,7 @@ def test_validate_constraints( constraint_violations = validate_storage_constraints( constraints=storage_device_constraints, soc_at_start=0.0, - min_soc=0, + soc_min=0, soc_max=10, resolution=resolution, ) From d70916f452a55621f1fadb6ec87be3ac6e945478 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Thu, 25 May 2023 22:50:45 +0200 Subject: [PATCH 19/19] feat: update validate_constraint to accept arbitrary constraint expressions Signed-off-by: Victor Garcia Reolid --- flexmeasures/data/models/planning/storage.py | 292 +++++++----------- .../data/models/planning/tests/test_solver.py | 28 +- 2 files changed, 132 insertions(+), 188 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index d3a6710a7..fa8bdce97 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1,5 +1,7 @@ from __future__ import annotations +import re +import copy from datetime import datetime, timedelta from typing import List, Dict @@ -52,7 +54,7 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: """Schedule a battery or Charge Point based directly on the latest beliefs regarding market prices within the specified time window. For the resulting consumption schedule, consumption is defined as positive values. - :param skip_validation: If True, skip validation of constraints specified in the data. + :param skip_validation: If True, skip validation of _constraints specified in the data. :returns: The computed schedule. """ if not self.config_deserialized: @@ -124,7 +126,7 @@ def compute(self, skip_validation: bool = False) -> pd.Series | None: down_deviation_prices.loc[start : end - resolution]["event_value"] ] - # Set up device constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n). + # Set up device _constraints: only one scheduled flexible device for this EMS (at index 0), plus the forecasted inflexible devices (at indices 1 to n). device_constraints = [ initialize_df(StorageScheduler.COLUMNS, start, end, resolution) for i in range(1 + len(inflexible_device_sensors)) @@ -450,7 +452,7 @@ def add_storage_constraints( :param soc_minima: Minimum state of charge at each time. :param soc_max: Maximum state of charge at all times. :param soc_min: Minimum state of charge at all times. - :returns: Constraints (StorageScheduler.COLUMNS) for a storage device, at each time step (index). + :returns: constraints (StorageScheduler.COLUMNS) for a storage device, at each time step (index). See device_scheduler for possible column names. """ @@ -523,12 +525,12 @@ def validate_storage_constraints( B.2) min <= equals B.3) equals <= max C. Validation in different time frames - C.1) equals(t) - equals(t-1) <= `derivative max`(t) - C.2) `derivative min`(t) <= equals(t) - equals(t-1) - C.3) min(t) - max(t-1) <= `derivative max`(t) - C.4) max(t) - min(t-1) >= `derivative min`(t) - C.5) condition equals(t) - max(t-1) <= `derivative max`(t) - C.6) `derivative min`(t) <= equals(t) - min(t-1) + C.1) equals(t) - equals(t-1) <= derivative_max(t) + C.2) derivative_min(t) <= equals(t) - equals(t-1) + C.3) min(t) - max(t-1) <= derivative_max(t) + C.4) max(t) - min(t-1) >= derivative_min(t) + C.5) condition equals(t) - max(t-1) <= derivative_max(t) + C.6) derivative_min(t) <= equals(t) - min(t-1) :param constraints: dataframe containing the constraints of a storage device :param soc_at_start: State of charge at the start time. @@ -538,241 +540,175 @@ def validate_storage_constraints( :returns: List of constraint violations, specifying their time, constraint and violation. """ + # get a copy of the constraints to make sure the dataframe doesn't get updated + _constraints = constraints.copy() + + _constraints = _constraints.rename( + columns={ + columns_name: columns_name.replace(" ", "_") + + "(t)" # replace spaces with underscore and add time index + for columns_name in _constraints.columns + } + ) + constraint_violations = [] + violations = validate_constraint(_constraints, "max(t) >= min(t)") + + print(violations) + ######################## # A. Global validation # ######################## # 1) min >= soc_min soc_min = (soc_min - soc_at_start) * timedelta(hours=1) / resolution - constraint_violations += validate_constraint( - constraints, - "min", - ">=", - "soc_min", - right_value=soc_min, - ) + _constraints["soc_min(t)"] = soc_min + constraint_violations += validate_constraint(_constraints, "soc_min(t) <= min(t)") # 2) max <= soc_max soc_max = (soc_max - soc_at_start) * timedelta(hours=1) / resolution - constraint_violations += validate_constraint( - constraints, - "max", - "<=", - "soc_max", - right_value=soc_max, - ) + _constraints["soc_max(t)"] = soc_max + constraint_violations += validate_constraint(_constraints, "max(t) <= soc_max(t)") ######################################## # B. Validation in the same time frame # ######################################## # 1) min <= max - constraint_violations += validate_constraint(constraints, "min", "<=", "max") + constraint_violations += validate_constraint(_constraints, "min(t) <= max(t)") # 2) min <= equals - constraint_violations += validate_constraint(constraints, "min", "<=", "equals") + constraint_violations += validate_constraint(_constraints, "min(t) <= equals(t)") # 3) equals <= max - constraint_violations += validate_constraint(constraints, "equals", "<=", "max") + constraint_violations += validate_constraint(_constraints, "equals(t) <= max(t)") ########################################## # C. Validation in different time frames # ########################################## - factor_w_wh = resolution / timedelta(hours=1) + _constraints["factor_w_wh(t)"] = resolution / timedelta(hours=1) + _constraints["min(t-1)"] = prepend_serie(_constraints["min(t)"], soc_min) + _constraints["equals(t-1)"] = prepend_serie(_constraints["equals(t)"], soc_at_start) + _constraints["max(t-1)"] = prepend_serie(_constraints["max(t)"], soc_max) - # compute diff_equals(t) = equals(t) - equals(t-1) - equals_extended = constraints["equals"].copy() - # insert `soc_at_start` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series - equals_extended[constraints.index[0] - resolution] = soc_at_start - # sort index to keep the time ordering - equals_extended = equals_extended.sort_index() + # 1) equals(t) - equals(t-1) <= derivative_max(t) + constraint_violations += validate_constraint( + _constraints, "equals(t) - equals(t-1) <= derivative_max(t) * factor_w_wh(t)" + ) - diff_equals = equals_extended.diff()[1:] + # 2) derivative_min(t) <= equals(t) - equals(t-1) + constraint_violations += validate_constraint( + _constraints, "derivative_min(t) * factor_w_wh(t) <= equals(t) - equals(t-1)" + ) - # 1) equals(t) - equals(t-1) <= `derivative max`(t) + # 3) min(t) - max(t-1) <= derivative_max(t) + constraint_violations += validate_constraint( + _constraints, "min(t) - max(t-1) <= derivative_max(t) * factor_w_wh(t)" + ) - mask = ( - ~(diff_equals <= constraints["derivative max"] * factor_w_wh) - & ~diff_equals.isna() + # 4) max(t) - min(t-1) >= derivative_min(t) + constraint_violations += validate_constraint( + _constraints, "derivative_min(t) * factor_w_wh(t) <= max(t) - min(t-1)" ) - time_condition_fails = constraints.index[mask] - for dt in time_condition_fails: - value_equals = constraints.loc[dt, "equals"] - value_equals_previous = constraints.loc[dt - resolution, "equals"] - value_derivative_max = constraints.loc[dt, "derivative max"] + # 5) equals(t) - max(t-1) <= derivative_max(t) + constraint_violations += validate_constraint( + _constraints, "equals(t) - max(t-1) <= derivative_max(t) * factor_w_wh(t)" + ) - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="equals(t) - equals(t-1) <= `derivative max`(t)", - violation=f"equals(t) [{value_equals}] - equals(t-1) [{value_equals_previous}] <= `derivative max`(t) [{value_derivative_max}]", - ) - ) + # 6) derivative_min(t) <= equals(t) - min(t-1) + constraint_violations += validate_constraint( + _constraints, "derivative_min(t) * factor_w_wh(t) <= equals(t) - min(t-1)" + ) - # 2) `derivative min`(t) <= equals(t) - equals(t-1) + return constraint_violations - mask = ( - ~((constraints["derivative min"] * factor_w_wh) <= diff_equals) - & ~diff_equals.isna() - ) - time_condition_fails = constraints.index[mask] - for dt in time_condition_fails: - value_equals = constraints.loc[dt, "equals"] - value_equals_previous = constraints.loc[dt - resolution, "equals"] - value_derivative_min = constraints.loc[dt, "derivative min"] +def get_pattern_match_word(word: str) -> str: + """Get a regex pattern to match a word - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="`derivative min`(t) <= equals(t) - equals(t-1)", - violation=f"`derivative min`(t) [{value_derivative_min}] <= equals(t) [{value_equals}] - equals(t-1) [{value_equals_previous}]", - ) - ) + The conditions to delimit a word are: + - start of line + - whitespace + - end of line + - word boundary + - arithmetic operations - # extend max - max_extended = constraints["max"].copy() - # insert `soc_max` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series - max_extended[constraints.index[0] - resolution] = soc_max - # sort index to keep the time ordering - max_extended = max_extended.sort_index() + :return: regex expression + """ - # extend min - min_extended = constraints["min"].copy() - # insert `soc_max` at time `constraints.index[0] - resolution` which creates a new entry at the end of the series - min_extended[constraints.index[0] - resolution] = soc_min - # sort index to keep the time ordering - min_extended = min_extended.sort_index() + regex = r"(^|\s|$|\b|\+|\-|\*|/\|\\)" - # 3) min(t) - max(t-1) <= `derivative max`(t) - delta_min_max = min_extended - max_extended.shift(1) - delta_min_max = delta_min_max[1:] + return regex + re.escape(word) + regex - condition3 = delta_min_max <= constraints["derivative max"] * factor_w_wh - mask = ~condition3 - time_condition_fails = constraints.index[mask] - for dt in time_condition_fails: - value_min = constraints.loc[dt, "min"] - value_max_previous = max_extended.loc[dt - resolution] - value_derivative_max = constraints.loc[dt, "derivative max"] +def validate_constraint( + constraints_df: pd.DataFrame, constraint_expression: str +) -> list[dict]: + """Validate the feasibility of a given set of constraints. - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="min(t) - max(t-1) <= `derivative max`(t)", - violation=f"min(t) [{value_min}] - max(t-1) [{value_max_previous}] <= `derivative max`(t) [{value_derivative_max}]", - ) - ) + :param constraints_df: DataFrame with the constraints + :param constraint_expression: inequality expression following pd.eval format. + No need to use the syntax `column` to reference + column, just use the column name. + :return: List of constraint violations, specifying their time, constraint and violation. + """ - # 4) max(t) - min(t-1) >= `derivative min`(t) - delta_max_min = max_extended - min_extended.shift(1) - delta_max_min = delta_max_min[1:] + columns_involved = [] - condition4 = delta_max_min >= constraints["derivative min"] * factor_w_wh - mask = ~condition4 - time_condition_fails = constraints.index[mask] + eval_expression = copy.copy(constraint_expression) - for dt in time_condition_fails: - value_max = constraints.loc[dt, "max"] - value_min_previous = min_extended.loc[dt - resolution] - value_derivative_min = constraints.loc[dt, "derivative min"] + for column in constraints_df.columns: + if re.search(get_pattern_match_word(column), eval_expression): + columns_involved.append(column) - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="max(t) - min(t-1) >= `derivative min`", - violation=f"max(t) [{value_max}] - min(t-1) [{value_min_previous}] >= `derivative min`(t) [{value_derivative_min}]", - ) + eval_expression = re.sub( + get_pattern_match_word(column), f"`{column}`", eval_expression ) - # 5) equals(t) - max(t-1) <= `derivative max`(t) - delta_equals_max = constraints["equals"] - max_extended.shift(1) - delta_equals_max = delta_equals_max[1:] + time_condition_fails = constraints_df.index[ + ~constraints_df.fillna(0).eval(eval_expression) + & ~constraints_df[columns_involved].isna().any(axis=1) + ] - condition5 = delta_equals_max <= constraints["derivative max"] * factor_w_wh - mask = ~condition5 & ~constraints["equals"].isna() - time_condition_fails = constraints.index[mask] + constraint_violations = [] for dt in time_condition_fails: - value_equals = constraints.loc[dt, "equals"] - value_max_previous = max_extended.loc[dt - resolution] - value_derivative_max = constraints.loc[dt, "derivative max"] + value_replaced = copy.copy(constraint_expression) - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition="equals(t) - max(t-1) <= `derivative max`(t)", - violation=f"equals(t) [{value_equals}] - max(t-1) [{value_max_previous}] <= `derivative max`(t) [{value_derivative_max}]", + for column in constraints_df.columns: + value_replaced = re.sub( + get_pattern_match_word(column), + f"{column} [{constraints_df.loc[dt, column]}]", + value_replaced, ) - ) - - # 6) `derivative min`(t) <= equals(t) - min(t-1) - delta_equals_min = constraints["equals"] - min_extended.shift(1) - delta_equals_min = delta_equals_min[1:] - - condition5 = delta_equals_min >= constraints["derivative min"] * factor_w_wh - mask = ~condition5 & ~constraints["equals"].isna() - time_condition_fails = constraints.index[mask] - - for dt in time_condition_fails: - value_equals = constraints.loc[dt, "equals"] - value_min_previous = min_extended.loc[dt - resolution] - value_derivative_min = constraints.loc[dt, "derivative min"] constraint_violations.append( dict( dt=dt.to_pydatetime(), - condition="`derivative min`(t) <= equals(t) - min(t-1)", - violation=f"`derivative min`(t) [{value_derivative_min}] <= equals(t) [{value_equals}] - min(t-1) [{value_min_previous}]", + condition=constraint_expression, + violation=value_replaced, ) ) return constraint_violations -def validate_constraint( - constraints, - left_constraint_name, - inequality, - right_constraint_name, - left_value: float | None = None, - right_value: float | None = None, -) -> list[dict]: - """Validate the feasibility of a given set of constraints. +def prepend_serie(serie: pd.Series, value) -> pd.Series: + """Prepend a value to a time series series - :returns: List of constraint violations, specifying their time, constraint and violation. + :param serie: serie containing the timed values + :param value: value to place in the first position """ - mask = True - if left_value is None: - left_value = constraints[left_constraint_name] - mask = mask & ~constraints[left_constraint_name].isna() - if right_value is None: - right_value = constraints[right_constraint_name] - mask = mask & ~constraints[right_constraint_name].isna() - if inequality == "<=": - mask = mask & ~(left_value <= right_value) - elif inequality == ">=": - mask = mask & ~(left_value >= right_value) - else: - raise NotImplementedError(f"Inequality '{inequality}' not supported.") - time_condition_fails = constraints.index[mask] - constraint_violations = [] - for dt in time_condition_fails: - lv = left_value[dt] if isinstance(left_value, pd.Series) else left_value - rv = right_value[dt] if isinstance(right_value, pd.Series) else right_value - constraint_violations.append( - dict( - dt=dt.to_pydatetime(), - condition=f"{left_constraint_name} {inequality} {right_constraint_name}", - violation=f"{left_constraint_name} [{lv}] {inequality} {right_constraint_name} [{rv}]", - ) - ) - return constraint_violations + # extend max + serie = serie.copy() + # insert `value` at time `serie.index[0] - resolution` which creates a new entry at the end of the series + serie[serie.index[0] - serie.index.freq] = value + # sort index to keep the time ordering + serie = serie.sort_index() + return serie.shift(1) ##################### diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index b644743a0..9baf56cb5 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -585,10 +585,10 @@ def test_add_storage_constraints( @pytest.mark.parametrize( "value_min1, value_equals1, value_max1, value_min2, value_equals2, value_max2, expected_constraint_type_violations", [ - (1, np.nan, 9, 2, np.nan, 20, ["max <= soc_max"]), - (-1, np.nan, 9, 1, np.nan, 9, ["min >= soc_min"]), - (1, 10, 9, 1, np.nan, 9, ["equals <= max"]), - (1, 0, 9, 1, np.nan, 9, ["min <= equals"]), + (1, np.nan, 9, 2, np.nan, 20, ["max(t) <= soc_max(t)"]), + (-1, np.nan, 9, 1, np.nan, 9, ["soc_min(t) <= min(t)"]), + (1, 10, 9, 1, np.nan, 9, ["equals(t) <= max(t)"]), + (1, 0, 9, 1, np.nan, 9, ["min(t) <= equals(t)"]), ( 1, np.nan, @@ -596,9 +596,17 @@ def test_add_storage_constraints( 9, np.nan, 1, - ["min <= max"], + ["min(t) <= max(t)"], + ), + ( + 9, + 5, + 1, + 1, + np.nan, + 9, + ["min(t) <= equals(t)", "equals(t) <= max(t)", "min(t) <= max(t)"], ), - (9, 5, 1, 1, np.nan, 9, ["min <= equals", "equals <= max", "min <= max"]), (1, np.nan, 9, 1, np.nan, 9, []), # same interval, should not fail (1, np.nan, 9, 3, np.nan, 7, []), # should not fail, containing interval (1, np.nan, 3, 3, np.nan, 5, []), # difference = 0 < 1, should not fail @@ -610,7 +618,7 @@ def test_add_storage_constraints( 5, np.nan, 7, - ["min(t) - max(t-1) <= `derivative max`(t)"], + ["min(t) - max(t-1) <= derivative_max(t) * factor_w_wh(t)"], ), # difference > max = 1, this should fail (3, np.nan, 5, 2, np.nan, 3, []), # difference = 0 < 1, should not fail (3, np.nan, 5, 1, np.nan, 2, []), # difference = -1 >= -1, should not fail @@ -621,7 +629,7 @@ def test_add_storage_constraints( 1, np.nan, 1, - ["max(t) - min(t-1) >= `derivative min`"], + ["derivative_min(t) * factor_w_wh(t) <= max(t) - min(t-1)"], ), # difference = -2 < -1, should fail, (1, 4, 9, 1, 4, 9, []), # same target value (4), should not fail ( @@ -631,7 +639,7 @@ def test_add_storage_constraints( 1, 4, 9, - ["`derivative min`(t) <= equals(t) - equals(t-1)"], + ["derivative_min(t) * factor_w_wh(t) <= equals(t) - equals(t-1)"], ), # difference = -2 < -1, should fail, ( 1, @@ -640,7 +648,7 @@ def test_add_storage_constraints( 1, 6, 9, - ["equals(t) - equals(t-1) <= `derivative max`(t)"], + ["equals(t) - equals(t-1) <= derivative_max(t) * factor_w_wh(t)"], ), # difference 2 > 1, should fail, ], )