Merge branch 'main' into 754-depricate-flask-env

.coveragerc

@@ -1,2 +1,2 @@
 [run]
-omit = */tests/*, */scripts/*, **/*.jinja, **/*.html, **/*.txt
+omit = */tests/*, */scripts/*, **/*.jinja, **/*.html, **/*.txt, flexmeasures/data/migrations/versions/*

documentation/api/notation.rst

@@ -195,6 +195,8 @@ Here are the three types of flexibility models you can expect to be built-in:
    - ``soc-maxima`` (defaults to NaN values)
    - ``soc-targets`` (defaults to NaN values)
    - ``roundtrip-efficiency`` (defaults to 100%)
+   - ``charging-efficiency`` (defaults to 100%)
+   - ``discharging-efficiency`` (defaults to 100%)
    - ``storage-efficiency`` (defaults to 100%) [#]_
    - ``prefer-charging-sooner`` (defaults to True, also signals a preference to discharge later)
    - ``power-capacity`` (defaults to the Sensor attribute ``capacity_in_mw``)
@@ -216,10 +218,9 @@ Here are the three types of flexibility models you can expect to be built-in:
 
    - Describe the thermal energy content in kWh or MWh.
    - Set ``soc-minima`` to the accumulative usage forecast.
-   - Set ``roundtrip-efficiency`` to the square of the conversion efficiency. [#]_
+   - Set ``charging-efficiency`` to the sensor describing the :abbr:`COP (coefficient of performance)` values.
    - Set ``storage-efficiency`` to a value below 100% to model (heat) loss.
 
-    .. [#] Setting a roundtrip efficiency of higher than 1 is not supported. We plan to implement a separate field for :abbr:`COP (coefficient of performance)` values.
 
 In addition, folks who write their own custom scheduler (see :ref:`plugin_customization`) might also require their custom flexibility model.
 That's no problem, FlexMeasures will let the scheduler decide which flexibility model is relevant and how it should be validated. 

documentation/changelog.rst

@@ -3,23 +3,40 @@
 FlexMeasures Changelog
 **********************
 
-v0.18.0 | December XX, 2023
+
+v0.19.0 | February xx, 2024
+============================
+
+New features
+-------------
+
+Infrastructure / Support
+----------------------
+
+Bugfixes
+-----------
+
+
+v0.18.0 | December 23, 2023
 ============================
 
-.. warning:: This version replaces FLASK_ENV with FLEXMEASURES_ENV (FLASK_ENV will still be used as a fallback).
+.. note:: Read more on these features on `the FlexMeasures blog <https://flexmeasures.io/018-better-use-of-future-knowledge/>`__.
+
 .. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``).
 
 New features
 -------------
 
 * Better navigation experience through listings (sensors / assets / users / accounts) in the :abbr:`UI (user interface)`, by heading to the selected entity upon a click (or CTRL + click) anywhere within a row [see `PR #923 <https://github.com/FlexMeasures/flexmeasures/pull/923>`_]
+* Introduce a breadcrumb to navigate through assets and sensor pages using its child-parent relationship [see `PR #930 <https://github.com/FlexMeasures/flexmeasures/pull/930>`_]
 * Define device-level power constraints as sensors to create schedules with changing power limits. [see `PR #897 <https://github.com/FlexMeasures/flexmeasures/pull/897>`_]
 * Allow to provide external storage usage or gain components using the ``soc-usage`` and ``soc-gain`` fields of the `flex-model` [see `PR #906 <https://github.com/FlexMeasures/flexmeasures/pull/906>`_]
+* Define time-varying charging and discharging efficiencies as sensors or as constant values which allows to define the :COP:`Coefficient of Performance`  [see `PR #933 <https://github.com/FlexMeasures/flexmeasures/pull/933>`_].
 
 Infrastructure / Support
 ----------------------
 
-* Deprecate use of flask's ``FLASK_ENV`` variable and replace it with ``FLEXMEASURES_ENV`` [see `PR #907 <https://github.com/FlexMeasures/flexmeasures/pull/907>`_]
+* Align database and models of `annotations`, `data_sources`, and `timed_belief` [see `PR #929 <https://github.com/FlexMeasures/flexmeasures/pull/929>`_]
 * New documentation section on constructing a flex model for :abbr:`V2G (vehicle-to-grid)` [see `PR #885 <https://github.com/FlexMeasures/flexmeasures/pull/885>`_]
 * Allow charts in plugins to show currency codes (such as EUR) as currency symbols (€) [see `PR #922 <https://github.com/FlexMeasures/flexmeasures/pull/922>`_]
 * Remove obsolete database tables `price`, `power`, `market`, `market_type`, `weather`, `asset`, and `weather_sensor` [see `PR #921 <https://github.com/FlexMeasures/flexmeasures/pull/921>`_]

documentation/concepts/device_scheduler.rst

@@ -75,7 +75,7 @@ The cost function quantifies the total cost of upwards and downwards deviations
 .. math:: 
     :name: cost_function
 
-    \min [\sum_{c,j} \Delta _{up}(c,j) \cdot Price_{up}(c,j) +  \Delta_{down}(c,j) \cdot Price_{down}(c,j)]
+    \min [\sum_{c,j} \Delta_{up}(c,j) \cdot Price_{up}(c,j) +  \Delta_{down}(c,j) \cdot Price_{down}(c,j)]
 
 
 State dynamics
@@ -191,5 +191,5 @@ Power coupling constraints
 .. math:: 
     :name: ems_flow_commitment_equalities
 
-    \sum_d P^{ems}(d,j) = \sum_c Commitment(c,j) + \Delta {up}(c,j) + \Delta {down}(c,j)
+    \sum_d P^{ems}(d,j) = \sum_c Commitment(c,j) + \Delta_{up}(c,j) + \Delta_{down}(c,j)
 

flexmeasures/api/v3_0/sensors.py

@@ -286,16 +286,19 @@ def trigger_schedule(
 
         This message triggers a 24-hour schedule for a storage asset, starting at 10.00am,
         at which the state of charge (soc) is 12.1 kWh, with a target state of charge of 25 kWh at 4.00pm.
+
+        The charging efficiency is constant (120%) and the discharging efficiency is determined by the contents of sensor
+        with id 98. If just the ``roundtrip-efficiency`` is known, it can be described with its own field.
         The global minimum and maximum soc are set to 10 and 25 kWh, respectively.
         To guarantee a minimum SOC in the period prior to 4.00pm, local minima constraints are imposed (via soc-minima)
         at 2.00pm and 3.00pm, for 15kWh and 20kWh, respectively.
-        Roundtrip efficiency for use in scheduling is set to 98%.
         Storage efficiency is set to 99.99%, denoting the state of charge left after each time step equal to the sensor's resolution.
         Aggregate consumption (of all devices within this EMS) should be priced by sensor 9,
         and aggregate production should be priced by sensor 10,
         where the aggregate power flow in the EMS is described by the sum over sensors 13, 14 and 15
         (plus the flexible sensor being optimized, of course).
 
+
         The battery consumption power capacity is limited by sensor 42 and the production capacity is constant (30 kW).
 
         Note that, if forecasts for sensors 13, 14 and 15 are not available, a schedule cannot be computed.
@@ -326,7 +329,8 @@ def trigger_schedule(
                     ],
                     "soc-min": 10,
                     "soc-max": 25,
-                    "roundtrip-efficiency": 0.98,
+                    "charging-efficiency": "120%",
+                    "discharging-efficiency": {"sensor" : 98},
                     "storage-efficiency": 0.9999,
                     "power-capacity": "25kW",
                     "consumption-capacity" : {"sensor" : 42},

flexmeasures/cli/data_add.py

@@ -67,6 +67,7 @@
 from flexmeasures.data.schemas.generic_assets import (
     GenericAssetSchema,
     GenericAssetTypeSchema,
+    GenericAssetIdField,
 )
 from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType
 from flexmeasures.data.models.user import User
@@ -369,6 +370,13 @@ def add_asset_type(**args):
     type=int,
     help="Asset type to assign to this asset",
 )
+@click.option(
+    "--parent-asset",
+    "parent_asset",
+    required=False,
+    type=GenericAssetIdField(),
+    help="Parent of this asset. The entity needs to exists on the database.",
+)
 def add_asset(**args):
     """Add an asset."""
     check_errors(GenericAssetSchema().validate(args))

flexmeasures/data/migrations/versions/c349f52c700d_update_data_sources.py

@@ -0,0 +1,73 @@
+"""Update data sources
+
+Revision ID: c349f52c700d
+Revises: ad98460751d9
+Create Date: 2023-12-14 10:31:02.612590
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "c349f52c700d"
+down_revision = "ad98460751d9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # The name of the data_source should be 120 String, this was not correctly set in an earlier revision of the db.
+    with op.batch_alter_table("data_source", schema=None) as batch_op:
+        batch_op.alter_column(
+            "name",
+            existing_type=sa.VARCHAR(length=80),
+            type_=sa.String(length=120),
+            existing_nullable=False,
+        )
+        # The attributes were initially set as nullable=False but the migration file did not reflect that.
+        # In this migration the model and db are brought in line.
+        batch_op.alter_column(
+            "attributes",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=False,
+        )
+
+    # This constraint is renamed to include the full name of the `data_source` table.
+    with op.batch_alter_table("timed_belief", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            "timed_belief_source_id_source_fkey", type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("timed_belief_source_id_data_source_fkey"),
+            "data_source",
+            ["source_id"],
+            ["id"],
+        )
+
+
+def downgrade():
+    with op.batch_alter_table("timed_belief", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("timed_belief_source_id_data_source_fkey"), type_="foreignkey"
+        )
+        batch_op.create_foreign_key(
+            "timed_belief_source_id_source_fkey",
+            "data_source",
+            ["source_id"],
+            ["id"],
+            ondelete="CASCADE",
+        )
+
+    with op.batch_alter_table("data_source", schema=None) as batch_op:
+        batch_op.alter_column(
+            "attributes",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=True,
+        )
+        batch_op.alter_column(
+            "name",
+            existing_type=sa.String(length=120),
+            type_=sa.VARCHAR(length=80),
+            existing_nullable=False,
+        )

flexmeasures/data/models/annotations.py

@@ -22,14 +22,15 @@ class Annotation(db.Model):
     start = db.Column(db.DateTime(timezone=True), nullable=False)
     end = db.Column(db.DateTime(timezone=True), nullable=False)
     belief_time = db.Column(db.DateTime(timezone=True), nullable=True)
-    source_id = db.Column(db.Integer, db.ForeignKey("data_source.id"))
+    source_id = db.Column(db.Integer, db.ForeignKey("data_source.id"), nullable=False)
     source = db.relationship(
         "DataSource",
         foreign_keys=[source_id],
         backref=db.backref("annotations", lazy=True),
     )
     type = db.Column(
-        db.Enum("alert", "holiday", "label", "feedback", name="annotation_type")
+        db.Enum("alert", "holiday", "label", "feedback", name="annotation_type"),
+        nullable=False,
     )
     content = db.Column(db.String(1024), nullable=False)
     __table_args__ = (

flexmeasures/data/models/planning/storage.py

@@ -9,6 +9,7 @@
 import numpy as np
 from flask import current_app
 
+
 from flexmeasures.data.models.planning import Scheduler, SchedulerOutputType
 from flexmeasures.data.models.planning.linear_optimization import device_scheduler
 from flexmeasures.data.models.planning.utils import (
@@ -100,7 +101,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
         soc_max = self.flex_model.get("soc_max")
         soc_minima = self.flex_model.get("soc_minima")
         soc_maxima = self.flex_model.get("soc_maxima")
-        roundtrip_efficiency = self.flex_model.get("roundtrip_efficiency")
         storage_efficiency = self.flex_model.get("storage_efficiency")
         prefer_charging_sooner = self.flex_model.get("prefer_charging_sooner", True)
 
@@ -266,10 +266,38 @@ def _prepare(self, skip_validation: bool = False) -> tuple:  # noqa: C901
             device_constraints[0]["stock delta"] *= timedelta(hours=1) / resolution
 
         # Apply round-trip efficiency evenly to charging and discharging
-        device_constraints[0]["derivative down efficiency"] = (
-            roundtrip_efficiency**0.5
+        charging_efficiency = get_continuous_series_sensor_or_quantity(
+            quantity_or_sensor=self.flex_model.get("charging_efficiency"),
+            actuator=sensor,
+            unit="dimensionless",
+            query_window=(start, end),
+            resolution=resolution,
+            beliefs_before=belief_time,
+            fallback_attribute="charging-efficiency",
+        ).fillna(1)
+        discharging_efficiency = get_continuous_series_sensor_or_quantity(
+            quantity_or_sensor=self.flex_model.get("discharging_efficiency"),
+            actuator=sensor,
+            unit="dimensionless",
+            query_window=(start, end),
+            resolution=resolution,
+            beliefs_before=belief_time,
+            fallback_attribute="discharging-efficiency",
+        ).fillna(1)
+
+        roundtrip_efficiency = self.flex_model.get(
+            "roundtrip_efficiency", self.sensor.get_attribute("roundtrip_efficiency", 1)
         )
-        device_constraints[0]["derivative up efficiency"] = roundtrip_efficiency**0.5
+
+        # if roundtrip efficiency is provided in the flex-model or defined as an asset attribute
+        if "roundtrip_efficiency" in self.flex_model or self.sensor.has_attribute(
+            "roundtrip-efficiency"
+        ):
+            charging_efficiency = roundtrip_efficiency**0.5
+            discharging_efficiency = roundtrip_efficiency**0.5
+
+        device_constraints[0]["derivative down efficiency"] = discharging_efficiency
+        device_constraints[0]["derivative up efficiency"] = charging_efficiency
 
         # Apply storage efficiency (accounts for losses over time)
         device_constraints[0]["efficiency"] = storage_efficiency
@@ -439,16 +467,6 @@ def deserialize_flex_config(self):
                 "storage_efficiency", 1
             )
 
-        # Check for round-trip efficiency
-        # todo: simplify to: `if self.flex_model.get("roundtrip-efficiency") is None:`
-        if (
-            "roundtrip-efficiency" not in self.flex_model
-            or self.flex_model["roundtrip-efficiency"] is None
-        ):
-            # Get default from sensor, or use 100% otherwise
-            self.flex_model["roundtrip-efficiency"] = self.sensor.get_attribute(
-                "roundtrip_efficiency", 1
-            )
         self.ensure_soc_min_max()
 
         # Now it's time to check if our flex configurations holds up to schemas

flexmeasures/data/models/planning/tests/conftest.py

@@ -219,6 +219,44 @@ def process(db, building, setup_sources) -> dict[str, Sensor]:
     return _process
 
 
+@pytest.fixture(scope="module")
+def efficiency_sensors(db, add_battery_assets, setup_sources) -> dict[str, Sensor]:
+    battery = add_battery_assets["Test battery"]
+    sensors = {}
+    sensor_specs = [("efficiency", timedelta(minutes=15), 90)]
+
+    for name, resolution, value in sensor_specs:
+        # 1 days of test data
+        time_slots = initialize_index(
+            start=pd.Timestamp("2015-01-01").tz_localize("Europe/Amsterdam"),
+            end=pd.Timestamp("2015-01-02").tz_localize("Europe/Amsterdam"),
+            resolution=resolution,
+        )
+
+        efficiency_sensor = Sensor(
+            name=name,
+            unit="%",
+            event_resolution=resolution,
+            generic_asset=battery,
+        )
+        db.session.add(efficiency_sensor)
+        db.session.flush()
+
+        steps_in_hour = int(timedelta(hours=1) / resolution)
+        efficiency = [value] * len(time_slots)
+
+        add_as_beliefs(
+            db,
+            efficiency_sensor,
+            efficiency[:-steps_in_hour],
+            time_slots[:-steps_in_hour],
+            setup_sources["Seita"],
+        )
+        sensors[name] = efficiency_sensor
+
+    return sensors
+
+
 @pytest.fixture(scope="module")
 def add_stock_delta(db, add_battery_assets, setup_sources) -> dict[str, Sensor]:
     """