diff --git a/documentation/changelog.rst b/documentation/changelog.rst index aff3d4851..49995d45a 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -8,7 +8,8 @@ v0.22.0 | June XX, 2024 New features ------------- -* On the asset page, facilitate comparison by showing the two default sensors together if they record the same unit [see `PR #1066 `_] +* Flex-context (price sensors and inflexible device sensors) can now be set on the asset page (and are part of GenericAsset model) [see `PR #1059 `_] +* On the asset page's default view, facilitate comparison by showing the two default sensors together if they record the same unit [see `PR #1066 `_] Infrastructure / Support ---------------------- diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 14c81edab..5067c5d52 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -140,8 +140,13 @@ def post(self, asset_data: dict): :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ + inflexible_sensor_ids = asset_data.pop("inflexible_device_sensor_ids", []) asset = GenericAsset(**asset_data) db.session.add(asset) + # assign asset id + db.session.flush() + + asset.set_inflexible_sensors(inflexible_sensor_ids) db.session.commit() return asset_schema.dump(asset), 201 @@ -231,6 +236,9 @@ def patch(self, asset_data: dict, id: int, db_asset: GenericAsset): :status 403: INVALID_SENDER :status 422: UNPROCESSABLE_ENTITY """ + inflexible_sensor_ids = asset_data.pop("inflexible_device_sensor_ids", []) + db_asset.set_inflexible_sensors(inflexible_sensor_ids) + for k, v in asset_data.items(): setattr(db_asset, k, v) db.session.add(db_asset) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index edb31eace..a90949705 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -221,6 +221,7 @@ def get_data(self, sensor_data_description: dict): ), "flex_model": fields.Dict(data_key="flex-model"), "flex_context": fields.Dict(required=False, data_key="flex-context"), + "force_new_job_creation": fields.Boolean(required=False), }, location="json", ) @@ -233,6 +234,7 @@ def trigger_schedule( belief_time: datetime | None = None, flex_model: dict | None = None, flex_context: dict | None = None, + force_new_job_creation: bool | None = False, **kwargs, ): """ @@ -376,6 +378,7 @@ def trigger_schedule( job = create_scheduling_job( **scheduler_kwargs, enqueue=True, + force_new_job_creation=force_new_job_creation, ) except ValidationError as err: return invalid_flex_config(err.messages) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index 77ebf15b5..908a20975 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -1,12 +1,18 @@ +from datetime import timedelta from flask import url_for import pytest from isodate import parse_datetime, parse_duration import pandas as pd from rq.job import Job +from unittest.mock import patch from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule -from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.models.generic_assets import ( + GenericAsset, + GenericAssetInflexibleSensorRelationship, +) +from flexmeasures.data.models.planning.utils import get_prices, get_power_values from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.tests.utils import work_on_rq from flexmeasures.data.services.scheduling import ( @@ -54,8 +60,7 @@ def test_trigger_and_get_schedule( sensor = ( Sensor.query.filter(Sensor.name == "power") - .join(GenericAsset) - .filter(GenericAsset.id == Sensor.generic_asset_id) + .join(GenericAsset, GenericAsset.id == Sensor.generic_asset_id) .filter(GenericAsset.name == asset_name) .one_or_none() ) @@ -193,3 +198,222 @@ def test_trigger_and_get_schedule( # Check whether the soc-at-start was persisted as an asset attribute assert sensor.generic_asset.get_attribute("soc_in_mwh") == start_soc + + +@pytest.mark.parametrize( + "context_sensor, asset_sensor, parent_sensor, expect_sensor", + [ + # Only context sensor present, use it + ("epex_da", None, None, "epex_da"), + # Only asset sensor present, use it + (None, "epex_da", None, "epex_da"), + # Have sensors both in context and on asset, use from context + ("epex_da_production", "epex_da", None, "epex_da_production"), + # No sensor in context or asset, use from parent asset + (None, None, "epex_da", "epex_da"), + # No sensor in context, have sensor on asset and parent asset, use from asset + (None, "epex_da", "epex_da_production", "epex_da"), + ], +) +@pytest.mark.parametrize( + "sensor_type", + [ + "consumption", + "production", + ], +) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_price_sensor_priority( + app, + fresh_db, + add_market_prices_fresh_db, + add_battery_assets_fresh_db, + battery_soc_sensor_fresh_db, + add_charging_station_assets_fresh_db, + keep_scheduling_queue_empty, + context_sensor, + asset_sensor, + parent_sensor, + expect_sensor, + sensor_type, + requesting_user, +): # noqa: C901 + message, asset_name = message_for_trigger_schedule(), "Test battery" + message["force_new_job_creation"] = True + + sensor_types = ["consumption", "production"] + other_sensors = { + name: other_name + for name, other_name in zip(sensor_types, reversed(sensor_types)) + } + used_sensor, unused_sensor = ( + f"{sensor_type}-price-sensor", + f"{other_sensors[sensor_type]}-price-sensor", + ) + + price_sensor_id = None + sensor_attribute = f"{sensor_type}_price_sensor_id" + # preparation: ensure the asset actually has the price sensor set as attribute + if asset_sensor: + price_sensor_id = add_market_prices_fresh_db[asset_sensor].id + battery_asset = add_battery_assets_fresh_db[asset_name] + setattr(battery_asset, sensor_attribute, price_sensor_id) + fresh_db.session.add(battery_asset) + if parent_sensor: + price_sensor_id = add_market_prices_fresh_db[parent_sensor].id + building_asset = add_battery_assets_fresh_db["Test building"] + setattr(building_asset, sensor_attribute, price_sensor_id) + fresh_db.session.add(building_asset) + + # Adding unused sensor to context (e.g consumption price sensor if we test production sensor) + message["flex-context"] = { + unused_sensor: add_market_prices_fresh_db["epex_da"].id, + "site-power-capacity": "1 TW", # should be big enough to avoid any infeasibilities + } + if context_sensor: + price_sensor_id = add_market_prices_fresh_db[context_sensor].id + message["flex-context"][used_sensor] = price_sensor_id + + # trigger a schedule through the /sensors//schedules/trigger [POST] api endpoint + assert len(app.queues["scheduling"]) == 0 + + sensor = ( + Sensor.query.filter(Sensor.name == "power") + .join(GenericAsset, GenericAsset.id == Sensor.generic_asset_id) + .filter(GenericAsset.name == asset_name) + .one_or_none() + ) + with app.test_client() as client: + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + print("Server responded with:\n%s" % trigger_schedule_response.json) + assert trigger_schedule_response.status_code == 200 + + with patch( + "flexmeasures.data.models.planning.storage.get_prices", wraps=get_prices + ) as mock_storage_get_prices: + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + expect_price_sensor_id = add_market_prices_fresh_db[expect_sensor].id + # get_prices is called twice: 1st call has consumption price sensor, 2nd call has production price sensor + call_num = 0 if sensor_type == "consumption" else 1 + call_args = mock_storage_get_prices.call_args_list[call_num] + assert call_args[1]["price_sensor"].id == expect_price_sensor_id + + +@pytest.mark.parametrize( + "context_sensor_num, asset_sensor_num, parent_sensor_num, expect_sensor_num", + [ + # Sensors are present in context and parent, use from context + (1, 0, 2, 1), + # No sensors in context, have in asset and parent, use asset sensors + (0, 1, 2, 1), + # No sensors in context and asset, use from parent asset + (0, 0, 1, 1), + # Have sensors everywhere, use from context + (1, 2, 3, 1), + ], +) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_inflexible_device_sensors_priority( + app, + fresh_db, + add_market_prices_fresh_db, + add_battery_assets_fresh_db, + battery_soc_sensor_fresh_db, + add_charging_station_assets_fresh_db, + keep_scheduling_queue_empty, + context_sensor_num, + asset_sensor_num, + parent_sensor_num, + expect_sensor_num, + requesting_user, +): # noqa: C901 + message, asset_name = message_for_trigger_schedule(), "Test battery" + message["force_new_job_creation"] = True + + price_sensor_id = add_market_prices_fresh_db["epex_da"].id + message["flex-context"] = { + "consumption-price-sensor": price_sensor_id, + "production-price-sensor": price_sensor_id, + "site-power-capacity": "1 TW", # should be big enough to avoid any infeasibilities + } + if context_sensor_num: + other_asset = add_battery_assets_fresh_db["Test small battery"] + context_sensors = setup_inflexible_device_sensors( + fresh_db, other_asset, "other asset senssors", context_sensor_num + ) + message["flex-context"]["inflexible-device-sensors"] = [ + sensor.id for sensor in context_sensors + ] + if asset_sensor_num: + battery_asset = add_battery_assets_fresh_db[asset_name] + battery_sensors = setup_inflexible_device_sensors( + fresh_db, battery_asset, "battery asset sensors", asset_sensor_num + ) + link_sensors(fresh_db, battery_asset, battery_sensors) + if parent_sensor_num: + building_asset = add_battery_assets_fresh_db["Test building"] + building_sensors = setup_inflexible_device_sensors( + fresh_db, building_asset, "building asset sensors", parent_sensor_num + ) + link_sensors(fresh_db, building_asset, building_sensors) + + # trigger a schedule through the /sensors//schedules/trigger [POST] api endpoint + assert len(app.queues["scheduling"]) == 0 + + sensor = ( + Sensor.query.filter(Sensor.name == "power") + .join(GenericAsset, GenericAsset.id == Sensor.generic_asset_id) + .filter(GenericAsset.name == asset_name) + .one_or_none() + ) + with app.test_client() as client: + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + print("Server responded with:\n%s" % trigger_schedule_response.json) + assert trigger_schedule_response.status_code == 200 + + with patch( + "flexmeasures.data.models.planning.storage.get_power_values", + wraps=get_power_values, + ) as mock_storage_get_power_values: + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + # Counting how many times power values (for inflexible sensors) were fetched (gives us the number of sensors) + call_args = mock_storage_get_power_values.call_args_list + assert len(call_args) == expect_sensor_num + + +def setup_inflexible_device_sensors(fresh_db, asset, sensor_name, sensor_num): + """Test helper function to add sensor_num sensors to an asset""" + sensors = list() + for i in range(sensor_num): + sensor = Sensor( + name=f"{sensor_name}-{i}", + generic_asset=asset, + event_resolution=timedelta(hours=1), + unit="MW", + attributes={"capacity_in_mw": 2}, + ) + fresh_db.session.add(sensor) + sensors.append(sensor) + fresh_db.session.flush() + + return sensors + + +def link_sensors(fresh_db, asset, sensors): + for sensor in sensors: + asset_inflexible_sensor_relationship = GenericAssetInflexibleSensorRelationship( + generic_asset_id=asset.id, inflexible_sensor_id=sensor.id + ) + fresh_db.session.add(asset_inflexible_sensor_relationship) diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index e9c3d83c8..d7a3cf4f1 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -658,6 +658,16 @@ def add_market_prices_common( source=setup_sources["Seita"], ) + add_beliefs( + db=db, + sensor=setup_markets["epex_da_production"], + time_slots=time_slots, + values=[ + random() * (1 + np.sin(x * 2 * np.pi / 24)) for x in range(len(time_slots)) + ], + source=setup_sources["Seita"], + ) + # another day of test data (8 expensive hours, 8 cheap hours, and again 8 expensive hours) time_slots = initialize_index( start=pd.Timestamp("2015-01-02").tz_localize("Europe/Amsterdam"), @@ -760,6 +770,19 @@ def create_test_battery_assets( """ Add two battery assets, set their capacity values and their initial SOC. """ + building_type = GenericAssetType(name="building") + db.session.add(building_type) + test_building = GenericAsset( + name="building", + generic_asset_type=building_type, + owner=setup_accounts["Prosumer"], + attributes=dict( + capacity_in_mw=2, + ), + ) + db.session.add(test_building) + db.session.flush() + battery_type = generic_asset_types["battery"] test_battery = GenericAsset( @@ -768,6 +791,7 @@ def create_test_battery_assets( generic_asset_type=battery_type, latitude=10, longitude=100, + parent_asset_id=test_building.id, attributes=dict( capacity_in_mw=2, max_soc_in_mwh=5, @@ -891,6 +915,7 @@ def create_test_battery_assets( db.session.flush() return { + "Test building": test_building, "Test battery": test_battery, "Test battery with no known prices": test_battery_no_prices, "Test small battery": test_small_battery, diff --git a/flexmeasures/data/migrations/versions/202505c5cb06_assets_inflexible_sensors_table_.py b/flexmeasures/data/migrations/versions/202505c5cb06_assets_inflexible_sensors_table_.py new file mode 100644 index 000000000..f5a83908d --- /dev/null +++ b/flexmeasures/data/migrations/versions/202505c5cb06_assets_inflexible_sensors_table_.py @@ -0,0 +1,85 @@ +"""assets_inflexible_sensors table + consumption_price_sensor, production_price_sensor and inflexible_device_sensors to generic_asset + +Revision ID: 202505c5cb06 +Revises: 81cbbf42357b +Create Date: 2024-05-20 22:23:05.406911 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "202505c5cb06" +down_revision = "81cbbf42357b" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "assets_inflexible_sensors", + sa.Column("generic_asset_id", sa.Integer(), nullable=False), + sa.Column("inflexible_sensor_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["generic_asset_id"], + ["generic_asset.id"], + name=op.f("assets_inflexible_sensors_generic_asset_id_generic_asset_fkey"), + ), + sa.ForeignKeyConstraint( + ["inflexible_sensor_id"], + ["sensor.id"], + name=op.f("assets_inflexible_sensors_inflexible_sensor_id_sensor_fkey"), + ), + sa.PrimaryKeyConstraint( + "generic_asset_id", + "inflexible_sensor_id", + name=op.f("assets_inflexible_sensors_pkey"), + ), + sa.UniqueConstraint( + "inflexible_sensor_id", + "generic_asset_id", + name="assets_inflexible_sensors_key", + ), + ) + with op.batch_alter_table("generic_asset", schema=None) as batch_op: + batch_op.add_column( + sa.Column("consumption_price_sensor_id", sa.Integer(), nullable=True) + ) + batch_op.add_column( + sa.Column("production_price_sensor_id", sa.Integer(), nullable=True) + ) + batch_op.create_foreign_key( + batch_op.f("generic_asset_production_price_sensor_id_sensor_fkey"), + "sensor", + ["production_price_sensor_id"], + ["id"], + ondelete="SET NULL", + ) + batch_op.create_foreign_key( + batch_op.f("generic_asset_consumption_price_sensor_id_sensor_fkey"), + "sensor", + ["consumption_price_sensor_id"], + ["id"], + ondelete="SET NULL", + ) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("generic_asset", schema=None) as batch_op: + batch_op.drop_constraint( + batch_op.f("generic_asset_consumption_price_sensor_id_sensor_fkey"), + type_="foreignkey", + ) + batch_op.drop_constraint( + batch_op.f("generic_asset_production_price_sensor_id_sensor_fkey"), + type_="foreignkey", + ) + batch_op.drop_column("production_price_sensor_id") + batch_op.drop_column("consumption_price_sensor_id") + + op.drop_table("assets_inflexible_sensors") + # ### end Alembic commands ### diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 0beb50147..7b67c1c73 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -42,6 +42,26 @@ def __repr__(self): return "" % (self.id, self.name) +class GenericAssetInflexibleSensorRelationship(db.Model): + """Links assets to inflexible sensors.""" + + __tablename__ = "assets_inflexible_sensors" + + generic_asset_id = db.Column( + db.Integer, db.ForeignKey("generic_asset.id"), primary_key=True + ) + inflexible_sensor_id = db.Column( + db.Integer, db.ForeignKey("sensor.id"), primary_key=True + ) + __table_args__ = ( + db.UniqueConstraint( + "inflexible_sensor_id", + "generic_asset_id", + name="assets_inflexible_sensors_key", + ), + ) + + class GenericAsset(db.Model, AuthModelMixin): """An asset is something that has economic value. @@ -87,12 +107,37 @@ class GenericAsset(db.Model, AuthModelMixin): backref=db.backref("generic_assets", lazy=True), ) + consumption_price_sensor_id = db.Column( + db.Integer, db.ForeignKey("sensor.id", ondelete="SET NULL"), nullable=True + ) + consumption_price_sensor = db.relationship( + "Sensor", + foreign_keys=[consumption_price_sensor_id], + backref=db.backref("assets_with_this_consumption_price_context", lazy=True), + ) + + production_price_sensor_id = db.Column( + db.Integer, db.ForeignKey("sensor.id", ondelete="SET NULL"), nullable=True + ) + production_price_sensor = db.relationship( + "Sensor", + foreign_keys=[production_price_sensor_id], + backref=db.backref("assets_with_this_production_price_context", lazy=True), + ) + # Many-to-many relationships annotations = db.relationship( "Annotation", secondary="annotations_assets", backref=db.backref("assets", lazy="dynamic"), ) + inflexible_device_sensors = db.relationship( + "Sensor", + secondary="assets_inflexible_sensors", + backref=db.backref( + "assets_considering_this_as_inflexible_sensor_in_scheduling", lazy="dynamic" + ), + ) def __acl__(self): """ @@ -215,6 +260,33 @@ def set_attribute(self, attribute: str, value): if self.has_attribute(attribute): self.attributes[attribute] = value + def get_consumption_price_sensor(self): + """Searches for consumption_price_sensor upwards on the asset tree""" + if self.consumption_price_sensor: + return self.consumption_price_sensor + if self.parent_asset: + return self.parent_asset.get_consumption_price_sensor() + return None + + def get_production_price_sensor(self): + """Searches for production_price_sensor upwards on the asset tree""" + if self.production_price_sensor: + return self.production_price_sensor + if self.parent_asset: + return self.parent_asset.get_production_price_sensor() + return None + + def get_inflexible_device_sensors(self): + """ + Searches for inflexible_device_sensors upwards on the asset tree + This search will stop once any sensors are found (will not aggregate towards the top of the tree) + """ + if self.inflexible_device_sensors: + return self.inflexible_device_sensors + if self.parent_asset: + return self.parent_asset.get_inflexible_device_sensors() + return [] + @property def has_power_sensors(self) -> bool: """True if at least one power sensor is attached""" @@ -624,6 +696,23 @@ def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]: # noqa start, end = get_timerange(sensor_ids) return dict(start=start, end=end) + def set_inflexible_sensors(self, inflexible_sensor_ids: list[int]) -> None: + """Set inflexible sensors for this asset. + + :param inflexible_sensor_ids: list of sensor ids + """ + from flexmeasures.data.models.time_series import Sensor + + # -1 choice corresponds to "--Select sensor id--" which means no sensor is selected + # and all linked sensors should be unlinked + if len(inflexible_sensor_ids) == 1 and inflexible_sensor_ids[0] == -1: + self.inflexible_device_sensors = [] + else: + self.inflexible_device_sensors = Sensor.query.filter( + Sensor.id.in_(inflexible_sensor_ids) + ).all() + db.session.add(self) + def create_generic_asset(generic_asset_type: str, **kwargs) -> GenericAsset: """Create a GenericAsset and assigns it an id. diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 0e05cac14..a30bec654 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -87,10 +87,17 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 storage_efficiency = self.flex_model.get("storage_efficiency") prefer_charging_sooner = self.flex_model.get("prefer_charging_sooner", True) - consumption_price_sensor = self.flex_context.get("consumption_price_sensor") - production_price_sensor = self.flex_context.get("production_price_sensor") - inflexible_device_sensors = self.flex_context.get( - "inflexible_device_sensors", [] + consumption_price_sensor = ( + self.flex_context.get("consumption_price_sensor") + or self.sensor.generic_asset.get_consumption_price_sensor() + ) + production_price_sensor = ( + self.flex_context.get("production_price_sensor") + or self.sensor.generic_asset.get_production_price_sensor() + ) + inflexible_device_sensors = ( + self.flex_context.get("inflexible_device_sensors") + or self.sensor.generic_asset.get_inflexible_device_sensors() ) # Check for required Sensor attributes diff --git a/flexmeasures/data/models/planning/tests/conftest.py b/flexmeasures/data/models/planning/tests/conftest.py index ea7f902db..67b6049f9 100644 --- a/flexmeasures/data/models/planning/tests/conftest.py +++ b/flexmeasures/data/models/planning/tests/conftest.py @@ -6,6 +6,8 @@ from timely_beliefs.sensors.func_store.knowledge_horizons import at_date import pandas as pd +from sqlalchemy import select + from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.models.time_series import Sensor, TimedBelief @@ -95,7 +97,12 @@ def building(db, setup_accounts, setup_markets) -> GenericAsset: """ Set up a building. """ - building_type = GenericAssetType(name="building") + building_type = db.session.execute( + select(GenericAssetType).filter_by(name="building") + ).scalar_one_or_none() + if not building_type: + # create_test_battery_assets might have created it already + building_type = GenericAssetType(name="battery") db.session.add(building_type) building = GenericAsset( name="building", diff --git a/flexmeasures/data/queries/sensors.py b/flexmeasures/data/queries/sensors.py index 1e5c96495..3336f65a6 100644 --- a/flexmeasures/data/queries/sensors.py +++ b/flexmeasures/data/queries/sensors.py @@ -46,7 +46,7 @@ def query_sensors_by_proximity( closest_sensor_query = ( select(Sensor) - .join(GenericAsset) + .join(GenericAsset, Sensor.generic_asset_id == GenericAsset.id) .filter(Sensor.generic_asset_id == GenericAsset.id) ) if generic_asset_type_name: diff --git a/flexmeasures/data/schemas/generic_assets.py b/flexmeasures/data/schemas/generic_assets.py index 745020556..82435c24b 100644 --- a/flexmeasures/data/schemas/generic_assets.py +++ b/flexmeasures/data/schemas/generic_assets.py @@ -45,6 +45,11 @@ class GenericAssetSchema(ma.SQLAlchemySchema): attributes = JSON(required=False) parent_asset_id = fields.Int(required=False, allow_none=True) child_assets = ma.Nested("GenericAssetSchema", many=True, dumb_only=True) + production_price_sensor_id = fields.Int(required=False, allow_none=True) + consumption_price_sensor_id = fields.Int(required=False, allow_none=True) + inflexible_device_sensor_ids = fields.List( + fields.Int, required=False, allow_none=True + ) class Meta: model = GenericAsset diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index f9662cbf9..4e124fc86 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -38,9 +38,9 @@ def get_sensors( account_ids = [account.id for account in account] else: account_ids = [account.id] - sensor_query = sensor_query.join(GenericAsset).filter( - Sensor.generic_asset_id == GenericAsset.id - ) + sensor_query = sensor_query.join( + GenericAsset, GenericAsset.id == Sensor.generic_asset_id + ).filter(Sensor.generic_asset_id == GenericAsset.id) if include_public_assets: sensor_query = sensor_query.filter( sa.or_( diff --git a/flexmeasures/data/tests/test_generic_assets.py b/flexmeasures/data/tests/test_generic_assets.py new file mode 100644 index 000000000..7fb7053da --- /dev/null +++ b/flexmeasures/data/tests/test_generic_assets.py @@ -0,0 +1,120 @@ +import pytest +from datetime import timedelta + +from flexmeasures.data.models.generic_assets import ( + GenericAsset, + GenericAssetInflexibleSensorRelationship, +) +from flexmeasures.data.models.time_series import Sensor +from timely_beliefs.sensors.func_store.knowledge_horizons import x_days_ago_at_y_oclock + + +@pytest.mark.parametrize( + "setup_data, sensors_argument, expect_sensors_len", + [ + # Both sensors linked, unlink them + ({"link_sensor1": True, "link_sensor2": True}, [-1], 0), + # 1 sensor is linked, adding another one + ({"link_sensor1": True}, [1, 2], 2), + # 1 sensor is linked, unlinking it + ({"link_sensor1": True}, [-1], 0), + # 1 sensor is linked, replacing it with another + ({"link_sensor1": True}, [2], 1), + # no sensors linked, adding both + ({}, [1, 2], 2), + ], +) +def test_set_inflexible_sensors( + db, + setup_generic_asset_types, + setup_accounts, + setup_data, + sensors_argument, + expect_sensors_len, +): + """Check set_inflexible_sensors works as expected.""" + + with NewAssetWithSensors( + db, setup_generic_asset_types, setup_accounts, setup_data + ) as new_asset_decorator: + ids_substitution = { + -1: -1, + 1: new_asset_decorator.price_sensor1.id, + 2: new_asset_decorator.price_sensor2.id, + } + sensors_argument = [ids_substitution[id] for id in sensors_argument] + new_asset_decorator.test_battery.set_inflexible_sensors(sensors_argument) + + assert ( + db.session.query(GenericAssetInflexibleSensorRelationship).count() + == expect_sensors_len + ) + + +class NewAssetWithSensors: + def __init__(self, db, setup_generic_asset_types, setup_accounts, setup_data): + self.db = db + self.setup_generic_asset_types = setup_generic_asset_types + self.setup_accounts = setup_accounts + self.setup_data = setup_data + self.test_battery = None + self.price_sensor1 = None + self.price_sensor2 = None + self.relationships = list() + + def __enter__(self): + self.test_battery = GenericAsset( + name="test battery", + generic_asset_type=self.setup_generic_asset_types["battery"], + owner=None, + attributes={"some-attribute": "some-value", "sensors_to_show": [1, 2]}, + ) + self.db.session.add(self.test_battery) + self.db.session.flush() + + for attribute_name, sensor_name in zip( + ("price_sensor1", "price_sensor2"), ("sensor1", "sensor2") + ): + sensor = Sensor( + name=sensor_name, + generic_asset=self.test_battery, + event_resolution=timedelta(hours=1), + unit="EUR/MWh", + knowledge_horizon=( + x_days_ago_at_y_oclock, + {"x": 1, "y": 12, "z": "Europe/Paris"}, + ), + attributes=dict( + daily_seasonality=True, + weekly_seasonality=True, + yearly_seasonality=True, + ), + ) + setattr(self, attribute_name, sensor) + self.db.session.add(sensor) + self.db.session.flush() + + for attribute, sensor in zip( + ("price_sensor1", "price_sensor2"), (self.price_sensor1, self.price_sensor2) + ): + if self.setup_data.get(attribute): + relationship = GenericAssetInflexibleSensorRelationship( + generic_asset_id=self.test_battery.id, + inflexible_sensor_id=sensor.id, + ) + self.relationships.append(relationship) + self.db.session.add(relationship) + self.db.session.commit() + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Delete sensors and test_battery + for relationship in self.relationships: + self.db.session.delete(relationship) + self.db.session.commit() + + self.db.session.delete(self.price_sensor1) + self.db.session.delete(self.price_sensor2) + self.db.session.delete(self.test_battery) + self.db.session.commit() diff --git a/flexmeasures/data/tests/test_scheduling_repeated_jobs.py b/flexmeasures/data/tests/test_scheduling_repeated_jobs.py index 7f681273d..31acab44b 100644 --- a/flexmeasures/data/tests/test_scheduling_repeated_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_repeated_jobs.py @@ -139,7 +139,7 @@ def test_hashing(db, app, add_charging_station_assets, setup_test_data): charging_station = db.session.execute( select(Sensor) .filter(Sensor.name == "power") - .join(GenericAsset) + .join(GenericAsset, Sensor.generic_asset_id == GenericAsset.id) .filter(GenericAsset.id == Sensor.generic_asset_id) .filter(GenericAsset.name == "Test charging stations") ).scalar_one_or_none() diff --git a/flexmeasures/ui/crud/assets/__init__.py b/flexmeasures/ui/crud/assets/__init__.py new file mode 100644 index 000000000..61d559038 --- /dev/null +++ b/flexmeasures/ui/crud/assets/__init__.py @@ -0,0 +1,10 @@ +from flexmeasures.ui.crud.assets.forms import AssetForm, NewAssetForm +from flexmeasures.ui.crud.assets.utils import ( + get_allowed_price_sensor_data, + get_allowed_inflexible_sensor_data, + process_internal_api_response, + user_can_create_assets, + user_can_delete, + get_assets_by_account, +) +from flexmeasures.ui.crud.assets.views import AssetCrudUI diff --git a/flexmeasures/ui/crud/assets/forms.py b/flexmeasures/ui/crud/assets/forms.py new file mode 100644 index 000000000..5a1b5cbf2 --- /dev/null +++ b/flexmeasures/ui/crud/assets/forms.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import copy + +from flask import current_app +from flask_security import current_user +from flask_wtf import FlaskForm +from sqlalchemy import select +from wtforms import ( + StringField, + DecimalField, + SelectField, + SelectMultipleField, + ValidationError, +) +from wtforms.validators import DataRequired, optional +from typing import Optional + +from flexmeasures.auth.policy import user_has_admin_access +from flexmeasures.data import db +from flexmeasures.data.models.generic_assets import ( + GenericAsset, + GenericAssetType, + GenericAssetInflexibleSensorRelationship, +) +from flexmeasures.data.models.user import Account +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.ui.crud.assets.utils import ( + get_allowed_price_sensor_data, + get_allowed_inflexible_sensor_data, +) + + +class AssetForm(FlaskForm): + """The default asset form only allows to edit the name and location.""" + + name = StringField("Name") + latitude = DecimalField( + "Latitude", + validators=[optional()], + places=None, + render_kw={"placeholder": "--Click the map or enter a latitude--"}, + ) + longitude = DecimalField( + "Longitude", + validators=[optional()], + places=None, + render_kw={"placeholder": "--Click the map or enter a longitude--"}, + ) + attributes = StringField("Other attributes (JSON)", default="{}") + production_price_sensor_id = SelectField( + "Production price sensor", coerce=int, validate_choice=False + ) + consumption_price_sensor_id = SelectField( + "Consumption price sensor", coerce=int, validate_choice=False + ) + inflexible_device_sensor_ids = SelectMultipleField( + "Inflexible device sensors", coerce=int, validate_choice=False + ) + + def validate_inflexible_device_sensor_ids(form, field): + if field.data and len(field.data) > 1 and -1 in field.data: + raise ValidationError( + "No sensor choice is not allowed together with sensor ids." + ) + + def validate_on_submit(self): + if ( + hasattr(self, "generic_asset_type_id") + and self.generic_asset_type_id.data == -1 + ): + self.generic_asset_type_id.data = ( + "" # cannot be coerced to int so will be flagged as invalid input + ) + if hasattr(self, "account_id") and self.account_id.data == -1: + del self.account_id # asset will be public + result = super().validate_on_submit() + if ( + hasattr(self, "production_price_sensor_id") + and self.production_price_sensor_id is not None + and self.production_price_sensor_id.data == -1 + ): + self.production_price_sensor_id.data = None + if ( + hasattr(self, "consumption_price_sensor_id") + and self.consumption_price_sensor_id is not None + and self.consumption_price_sensor_id.data == -1 + ): + self.consumption_price_sensor_id.data = None + return result + + def to_json(self) -> dict: + """turn form data into a JSON we can POST to our internal API""" + data = copy.copy(self.data) + if data.get("longitude") is not None: + data["longitude"] = float(data["longitude"]) + if data.get("latitude") is not None: + data["latitude"] = float(data["latitude"]) + + if "csrf_token" in data: + del data["csrf_token"] + + return data + + def process_api_validation_errors(self, api_response: dict): + """Process form errors from the API for the WTForm""" + if not isinstance(api_response, dict): + return + for error_header in ("json", "validation_errors"): + if error_header not in api_response: + continue + for field in list(self._fields.keys()): + if field in list(api_response[error_header].keys()): + field_errors = api_response[error_header][field] + if isinstance(field_errors, list): + self._fields[field].errors += api_response[error_header][field] + else: + self._fields[field].errors.append( + api_response[error_header][field] + ) + + def with_options(self): + if "generic_asset_type_id" in self: + self.generic_asset_type_id.choices = [(-1, "--Select type--")] + [ + (atype.id, atype.name) + for atype in db.session.scalars(select(GenericAssetType)).all() + ] + if "account_id" in self: + self.account_id.choices = [(-1, "--Select account--")] + [ + (account.id, account.name) + for account in db.session.scalars(select(Account)).all() + ] + + def with_price_senors(self, asset: GenericAsset, account_id: Optional[int]) -> None: + allowed_price_sensor_data = get_allowed_price_sensor_data(account_id) + for sensor_name in ("production_price", "consumption_price"): + sensor_id = getattr(asset, sensor_name + "_sensor_id") if asset else None + if sensor_id: + sensor = Sensor.query.get(sensor_id) + allowed_price_sensor_data[sensor_id] = f"{asset.name}:{sensor.name}" + choices = [(id, label) for id, label in allowed_price_sensor_data.items()] + choices = [(-1, "--Select sensor id--")] + choices + form_sensor = getattr(self, sensor_name + "_sensor_id") + form_sensor.choices = choices + if sensor_id is None: + form_sensor.default = (-1, "--Select sensor id--") + else: + form_sensor.default = (sensor_id, allowed_price_sensor_data[sensor_id]) + setattr(self, sensor_name + "_sensor_id", form_sensor) + + def with_inflexible_sensors( + self, asset: GenericAsset, account_id: Optional[int] + ) -> None: + allowed_inflexible_sensor_data = get_allowed_inflexible_sensor_data(account_id) + linked_sensor_data = {} + if asset: + linked_sensors = ( + db.session.query( + GenericAssetInflexibleSensorRelationship.inflexible_sensor_id, + Sensor.name, + ) + .join( + Sensor, + GenericAssetInflexibleSensorRelationship.inflexible_sensor_id + == Sensor.id, + ) + .filter( + GenericAssetInflexibleSensorRelationship.generic_asset_id + == asset.id + ) + .all() + ) + linked_sensor_data = { + sensor_id: f"{asset.name}:{sensor_name}" + for sensor_id, sensor_name in linked_sensors + } + + all_sensor_data = {**allowed_inflexible_sensor_data, **linked_sensor_data} + choices = [(id, label) for id, label in all_sensor_data.items()] + choices = [(-1, "--Select sensor id--")] + choices + self.inflexible_device_sensor_ids.choices = choices + + default = [-1] + if linked_sensor_data: + default = [id for id in linked_sensor_data] + self.inflexible_device_sensor_ids.default = default + + def with_sensors( + self, + asset: GenericAsset, + account_id: Optional[int], + ) -> None: + self.with_price_senors(asset, account_id) + self.with_inflexible_sensors(asset, account_id) + + +class NewAssetForm(AssetForm): + """Here, in addition, we allow to set asset type and account.""" + + generic_asset_type_id = SelectField( + "Asset type", coerce=int, validators=[DataRequired()] + ) + account_id = SelectField("Account", coerce=int) + + def set_account(self) -> tuple[Account | None, str | None]: + """Set an account for the to-be-created asset. + Return the account (if available) and an error message""" + account_error = None + + if self.account_id.data == -1: + if user_has_admin_access(current_user, "update"): + return None, None # Account can be None (public asset) + else: + account_error = "Please pick an existing account." + + account = db.session.execute( + select(Account).filter_by(id=int(self.account_id.data)) + ).scalar_one_or_none() + + if account: + self.account_id.data = account.id + else: + current_app.logger.error(account_error) + return account, account_error + + def set_asset_type(self) -> tuple[GenericAssetType | None, str | None]: + """Set an asset type for the to-be-created asset. + Return the asset type (if available) and an error message.""" + asset_type = None + asset_type_error = None + + if int(self.generic_asset_type_id.data) == -1: + asset_type_error = "Pick an existing asset type." + else: + asset_type = db.session.execute( + select(GenericAssetType).filter_by( + id=int(self.generic_asset_type_id.data) + ) + ).scalar_one_or_none() + + if asset_type: + self.generic_asset_type_id.data = asset_type.id + else: + current_app.logger.error(asset_type_error) + return asset_type, asset_type_error diff --git a/flexmeasures/ui/crud/assets/utils.py b/flexmeasures/ui/crud/assets/utils.py new file mode 100644 index 000000000..2864b57c0 --- /dev/null +++ b/flexmeasures/ui/crud/assets/utils.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +import json +from flask import url_for +from flask_security import current_user +from typing import Optional, Dict +from sqlalchemy import select +from sqlalchemy.sql.expression import or_ + +from flexmeasures.auth.policy import check_access +from flexmeasures.data import db +from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType +from flexmeasures.data.models.user import Account +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.ui.crud.api_wrapper import InternalApi +from flexmeasures.utils.unit_utils import ( + is_energy_price_unit, + is_energy_unit, + is_power_unit, +) + + +def get_allowed_price_sensor_data(account_id: Optional[int]) -> Dict[int, str]: + """ + Return a list of sensors which the user can add + as consumption_price_sensor_id or production_price_sensor_id. + For each sensor we get data as sensor_id: asset_name:sensor_name. + """ + if not account_id: + assets = db.session.scalars( + select(GenericAsset).filter(GenericAsset.account_id.is_(None)) + ).all() + else: + assets = db.session.scalars( + select(GenericAsset).filter( + or_( + GenericAsset.account_id == account_id, + GenericAsset.account_id.is_(None), + ) + ) + ).all() + + sensors_data = list() + for asset in assets: + sensors_data += [ + (sensor.id, asset.name, sensor.name, sensor.unit) + for sensor in asset.sensors + ] + + return { + sensor_id: f"{asset_name}:{sensor_name}" + for sensor_id, asset_name, sensor_name, sensor_unit in sensors_data + if is_energy_price_unit(sensor_unit) + } + + +def get_allowed_inflexible_sensor_data(account_id: Optional[int]) -> Dict[int, str]: + """ + Return a list of sensors which the user can add + as inflexible device sensors. + This list is built using sensors with energy or power units + within the current account (or among public assets when account_id argument is not specified). + For each sensor we get data as sensor_id: asset_name:sensor_name. + """ + query = None + if not account_id: + query = select(GenericAsset).filter(GenericAsset.account_id.is_(None)) + else: + query = select(GenericAsset).filter(GenericAsset.account_id == account_id) + assets = db.session.scalars(query).all() + + sensors_data = list() + for asset in assets: + sensors_data += [ + (sensor.id, asset.name, sensor.name, sensor.unit) + for sensor in asset.sensors + ] + + return { + sensor_id: f"{asset_name}:{sensor_name}" + for sensor_id, asset_name, sensor_name, sensor_unit in sensors_data + if is_energy_unit(sensor_unit) or is_power_unit(sensor_unit) + } + + +def process_internal_api_response( + asset_data: dict, asset_id: int | None = None, make_obj=False +) -> GenericAsset | dict: + """ + Turn data from the internal API into something we can use to further populate the UI. + Either as an asset object or a dict for form filling. + + If we add other data by querying the database, we make sure the asset is not in the session afterwards. + """ + + def expunge_asset(): + # use if no insert is wanted from a previous query which flushes its results + if asset in db.session: + db.session.expunge(asset) + + asset_data.pop("status", None) # might have come from requests.response + if asset_id: + asset_data["id"] = asset_id + if make_obj: + children = asset_data.pop("child_assets", []) + + asset = GenericAsset( + **{ + **asset_data, + **{"attributes": json.loads(asset_data.get("attributes", "{}"))}, + } + ) # TODO: use schema? + asset.generic_asset_type = db.session.get( + GenericAssetType, asset.generic_asset_type_id + ) + expunge_asset() + asset.owner = db.session.get(Account, asset_data["account_id"]) + expunge_asset() + db.session.flush() + if "id" in asset_data: + asset.sensors = db.session.scalars( + select(Sensor).filter_by(generic_asset_id=asset_data["id"]) + ).all() + expunge_asset() + if asset_data.get("parent_asset_id", None) is not None: + asset.parent_asset = db.session.execute( + select(GenericAsset).filter( + GenericAsset.id == asset_data["parent_asset_id"] + ) + ).scalar_one_or_none() + expunge_asset() + + child_assets = [] + for child in children: + child.pop("child_assets") + child_asset = process_internal_api_response(child, child["id"], True) + child_assets.append(child_asset) + asset.child_assets = child_assets + expunge_asset() + + return asset + return asset_data + + +def user_can_create_assets() -> bool: + try: + check_access(current_user.account, "create-children") + except Exception: + return False + return True + + +def user_can_delete(asset) -> bool: + try: + check_access(asset, "delete") + except Exception: + return False + return True + + +def get_assets_by_account(account_id: int | str | None) -> list[GenericAsset]: + if account_id is not None: + get_assets_response = InternalApi().get( + url_for("AssetAPI:index"), query={"account_id": account_id} + ) + else: + get_assets_response = InternalApi().get(url_for("AssetAPI:public")) + return [ + process_internal_api_response(ad, make_obj=True) + for ad in get_assets_response.json() + ] diff --git a/flexmeasures/ui/crud/assets.py b/flexmeasures/ui/crud/assets/views.py similarity index 57% rename from flexmeasures/ui/crud/assets.py rename to flexmeasures/ui/crud/assets/views.py index 64e50bbb2..b4cef73d6 100644 --- a/flexmeasures/ui/crud/assets.py +++ b/flexmeasures/ui/crud/assets/views.py @@ -1,32 +1,28 @@ -from __future__ import annotations - -import copy -import json - from flask import url_for, current_app, request from flask_classful import FlaskView, route -from flask_wtf import FlaskForm from flask_security import login_required, current_user from webargs.flaskparser import use_kwargs -from wtforms import StringField, DecimalField, SelectField -from wtforms.validators import DataRequired, optional from sqlalchemy import select from flexmeasures.auth.policy import user_has_admin_access from flexmeasures.data import db from flexmeasures.auth.error_handling import unauthorized_handler -from flexmeasures.auth.policy import check_access from flexmeasures.data.schemas import StartEndTimeSchema from flexmeasures.data.services.job_cache import NoRedisConfigured from flexmeasures.data.models.generic_assets import ( - GenericAssetType, GenericAsset, get_center_location_of_assets, ) from flexmeasures.data.models.user import Account -from flexmeasures.data.models.time_series import Sensor from flexmeasures.ui.utils.view_utils import render_flexmeasures_template from flexmeasures.ui.crud.api_wrapper import InternalApi +from flexmeasures.ui.crud.assets.forms import NewAssetForm, AssetForm +from flexmeasures.ui.crud.assets.utils import ( + process_internal_api_response, + user_can_create_assets, + user_can_delete, + get_assets_by_account, +) from flexmeasures.data.services.sensors import ( build_sensor_status_data, build_asset_jobs_data, @@ -42,178 +38,6 @@ """ -class AssetForm(FlaskForm): - """The default asset form only allows to edit the name and location.""" - - name = StringField("Name") - latitude = DecimalField( - "Latitude", - validators=[optional()], - places=None, - render_kw={"placeholder": "--Click the map or enter a latitude--"}, - ) - longitude = DecimalField( - "Longitude", - validators=[optional()], - places=None, - render_kw={"placeholder": "--Click the map or enter a longitude--"}, - ) - attributes = StringField("Other attributes (JSON)", default="{}") - - def validate_on_submit(self): - if ( - hasattr(self, "generic_asset_type_id") - and self.generic_asset_type_id.data == -1 - ): - self.generic_asset_type_id.data = ( - "" # cannot be coerced to int so will be flagged as invalid input - ) - if hasattr(self, "account_id") and self.account_id.data == -1: - del self.account_id # asset will be public - return super().validate_on_submit() - - def to_json(self) -> dict: - """turn form data into a JSON we can POST to our internal API""" - data = copy.copy(self.data) - if data.get("longitude") is not None: - data["longitude"] = float(data["longitude"]) - if data.get("latitude") is not None: - data["latitude"] = float(data["latitude"]) - - if "csrf_token" in data: - del data["csrf_token"] - - return data - - def process_api_validation_errors(self, api_response: dict): - """Process form errors from the API for the WTForm""" - if not isinstance(api_response, dict): - return - for error_header in ("json", "validation_errors"): - if error_header not in api_response: - continue - for field in list(self._fields.keys()): - if field in list(api_response[error_header].keys()): - field_errors = api_response[error_header][field] - if isinstance(field_errors, list): - self._fields[field].errors += api_response[error_header][field] - else: - self._fields[field].errors.append( - api_response[error_header][field] - ) - - -class NewAssetForm(AssetForm): - """Here, in addition, we allow to set asset type and account.""" - - generic_asset_type_id = SelectField( - "Asset type", coerce=int, validators=[DataRequired()] - ) - account_id = SelectField("Account", coerce=int) - - -def with_options(form: AssetForm | NewAssetForm) -> AssetForm | NewAssetForm: - if "generic_asset_type_id" in form: - form.generic_asset_type_id.choices = [(-1, "--Select type--")] + [ - (atype.id, atype.name) - for atype in db.session.scalars(select(GenericAssetType)).all() - ] - if "account_id" in form: - form.account_id.choices = [(-1, "--Select account--")] + [ - (account.id, account.name) - for account in db.session.scalars(select(Account)).all() - ] - return form - - -def process_internal_api_response( - asset_data: dict, asset_id: int | None = None, make_obj=False -) -> GenericAsset | dict: - """ - Turn data from the internal API into something we can use to further populate the UI. - Either as an asset object or a dict for form filling. - - If we add other data by querying the database, we make sure the asset is not in the session afterwards. - """ - - def expunge_asset(): - # use if no insert is wanted from a previous query which flushes its results - if asset in db.session: - db.session.expunge(asset) - - asset_data.pop("status", None) # might have come from requests.response - if asset_id: - asset_data["id"] = asset_id - if make_obj: - children = asset_data.pop("child_assets", []) - - asset = GenericAsset( - **{ - **asset_data, - **{"attributes": json.loads(asset_data.get("attributes", "{}"))}, - } - ) # TODO: use schema? - asset.generic_asset_type = db.session.get( - GenericAssetType, asset.generic_asset_type_id - ) - expunge_asset() - asset.owner = db.session.get(Account, asset_data["account_id"]) - expunge_asset() - db.session.flush() - if "id" in asset_data: - asset.sensors = db.session.scalars( - select(Sensor).filter_by(generic_asset_id=asset_data["id"]) - ).all() - expunge_asset() - if asset_data.get("parent_asset_id", None) is not None: - asset.parent_asset = db.session.execute( - select(GenericAsset).filter( - GenericAsset.id == asset_data["parent_asset_id"] - ) - ).scalar_one_or_none() - expunge_asset() - - child_assets = [] - for child in children: - child.pop("child_assets") - child_asset = process_internal_api_response(child, child["id"], True) - child_assets.append(child_asset) - asset.child_assets = child_assets - expunge_asset() - - return asset - return asset_data - - -def user_can_create_assets() -> bool: - try: - check_access(current_user.account, "create-children") - except Exception: - return False - return True - - -def user_can_delete(asset) -> bool: - try: - check_access(asset, "delete") - except Exception: - return False - return True - - -def get_assets_by_account(account_id: int | str | None) -> list[GenericAsset]: - if account_id is not None: - get_assets_response = InternalApi().get( - url_for("AssetAPI:index"), query={"account_id": account_id} - ) - else: - get_assets_response = InternalApi().get(url_for("AssetAPI:public")) - return [ - process_internal_api_response(ad, make_obj=True) - for ad in get_assets_response.json() - ] - - class AssetCrudUI(FlaskView): """ These views help us offer a Jinja2-based UI. @@ -285,7 +109,9 @@ def get(self, id: str, **kwargs): if not user_can_create_assets(): return unauthorized_handler(None, []) - asset_form = with_options(NewAssetForm()) + asset_form = NewAssetForm() + asset_form.with_options() + asset_form.with_sensors(None, None) return render_flexmeasures_template( "crud/asset_new.html", asset_form=asset_form, @@ -296,10 +122,12 @@ def get(self, id: str, **kwargs): get_asset_response = InternalApi().get(url_for("AssetAPI:fetch_one", id=id)) asset_dict = get_asset_response.json() + asset = process_internal_api_response(asset_dict, int(id), make_obj=True) - asset_form = with_options(AssetForm()) + asset_form = AssetForm() + asset_form.with_options() + asset_form.with_sensors(asset, asset.account_id) - asset = process_internal_api_response(asset_dict, int(id), make_obj=True) asset_form.process(data=process_internal_api_response(asset_dict)) return render_flexmeasures_template( @@ -359,10 +187,14 @@ def post(self, id: str): error_msg = "" if id == "create": - asset_form = with_options(NewAssetForm()) + asset_form = NewAssetForm() + asset_form.with_options() + + account, account_error = asset_form.set_account() + asset_type, asset_type_error = asset_form.set_asset_type() - account, account_error = _set_account(asset_form) - asset_type, asset_type_error = _set_asset_type(asset_form) + account_id = account.id if account else None + asset_form.with_sensors(None, account_id) form_valid = asset_form.validate_on_submit() @@ -411,11 +243,16 @@ def post(self, id: str): ) else: - asset_form = with_options(AssetForm()) + asset = db.session.get(GenericAsset, id) + asset_form = AssetForm() + asset_form.with_options() + asset_form.with_sensors(asset, asset.account_id) if not asset_form.validate_on_submit(): - asset = db.session.get(GenericAsset, id) # Display the form data, but set some extra data which the page wants to show. asset_info = asset_form.to_json() + asset_info = { + k: v for k, v in asset_info.items() if k not in asset_form.errors + } asset_info["id"] = id asset_info["account_id"] = asset.account_id asset = process_internal_api_response( @@ -469,49 +306,3 @@ def delete_with_data(self, id: str): return self.index( msg=f"Asset {id} and assorted meter readings / forecasts have been deleted." ) - - -def _set_account(asset_form: NewAssetForm) -> tuple[Account | None, str | None]: - """Set an account for the to-be-created asset. - Return the account (if available) and an error message""" - account_error = None - - if asset_form.account_id.data == -1: - if user_has_admin_access(current_user, "update"): - return None, None # Account can be None (public asset) - else: - account_error = "Please pick an existing account." - - account = db.session.execute( - select(Account).filter_by(id=int(asset_form.account_id.data)) - ).scalar_one_or_none() - - if account: - asset_form.account_id.data = account.id - else: - current_app.logger.error(account_error) - return account, account_error - - -def _set_asset_type( - asset_form: NewAssetForm, -) -> tuple[GenericAssetType | None, str | None]: - """Set an asset type for the to-be-created asset. - Return the asset type (if available) and an error message.""" - asset_type = None - asset_type_error = None - - if int(asset_form.generic_asset_type_id.data) == -1: - asset_type_error = "Pick an existing asset type." - else: - asset_type = db.session.execute( - select(GenericAssetType).filter_by( - id=int(asset_form.generic_asset_type_id.data) - ) - ).scalar_one_or_none() - - if asset_type: - asset_form.generic_asset_type_id.data = asset_type.id - else: - current_app.logger.error(asset_type_error) - return asset_type, asset_type_error diff --git a/flexmeasures/ui/static/css/flexmeasures.css b/flexmeasures/ui/static/css/flexmeasures.css index 7e0b60117..f750c4f3b 100644 --- a/flexmeasures/ui/static/css/flexmeasures.css +++ b/flexmeasures/ui/static/css/flexmeasures.css @@ -1462,6 +1462,12 @@ body .dataTables_wrapper .dataTables_paginate .paginate_button.current:hover { margin-top: 30px; } +.asset-form .form-group-bordered { + border: 2px solid rgb(169, 176, 176); + padding: 10px; + margin-bottom: 10px; +} + /* ---- End Asset Form ---- */ diff --git a/flexmeasures/ui/templates/crud/asset.html b/flexmeasures/ui/templates/crud/asset.html index de5c7adac..e70cd18c8 100644 --- a/flexmeasures/ui/templates/crud/asset.html +++ b/flexmeasures/ui/templates/crud/asset.html @@ -122,6 +122,42 @@

Edit {{ asset.name }}

disabled> +
+

Fields in this section can form part of this asset's flex context. They can also be set in a parent asset or in API requests that trigger schedules.

+ {% if asset_form.consumption_price_sensor_id %} +
+ {{ asset_form.consumption_price_sensor_id.label(class="col-sm-3 control-label") }} +
+ {{ asset_form.consumption_price_sensor_id(class_="form-control") }} + {% for error in asset_form.errors.consumption_price_sensor_id %} + [{{error}}] + {% endfor %} +
+
+ {% endif %} + {% if asset_form.production_price_sensor_id %} +
+ {{ asset_form.production_price_sensor_id.label(class="col-sm-3 control-label") }} +
+ {{ asset_form.production_price_sensor_id(class_="form-control") }} + {% for error in asset_form.errors.production_price_sensor_id %} + [{{error}}] + {% endfor %} +
+
+ {% endif %} + {% if asset_form.inflexible_device_sensor_ids %} +
+ {{ asset_form.inflexible_device_sensor_ids.label(class="col-sm-3 control-label") }} +
+ {{ asset_form.inflexible_device_sensor_ids(class_="form-control") }} + {% for error in asset_form.errors.inflexible_device_sensor_ids %} + [{{error}}] + {% endfor %} +
+
+ {% endif %} +
{{ asset_form.attributes.label(class="col-sm-3 control-label") }}
diff --git a/flexmeasures/ui/templates/crud/asset_new.html b/flexmeasures/ui/templates/crud/asset_new.html index 2ed242221..b7c094235 100644 --- a/flexmeasures/ui/templates/crud/asset_new.html +++ b/flexmeasures/ui/templates/crud/asset_new.html @@ -70,6 +70,40 @@

Creating a new asset

{% endfor %}
+

Fields below can form part of this asset's flex context (flex context). They can also be set in a parent asset or in the API requests.

+ {% if asset_form.consumption_price_sensor_id %} +
+ {{ asset_form.consumption_price_sensor_id.label(class="col-sm-3 control-label") }} +
+ {{ asset_form.consumption_price_sensor_id(class_="form-control") }} + {% for error in asset_form.errors.consumption_price_sensor_id %} + [{{error}}] + {% endfor %} +
+
+ {% endif %} + {% if asset_form.production_price_sensor_id %} +
+ {{ asset_form.production_price_sensor_id.label(class="col-sm-3 control-label") }} +
+ {{ asset_form.production_price_sensor_id(class_="form-control") }} + {% for error in asset_form.errors.production_price_sensor_id %} + [{{error}}] + {% endfor %} +
+
+ {% endif %} + {% if asset_form.inflexible_device_sensor_ids %} +
+ {{ asset_form.inflexible_device_sensor_ids.label(class="col-sm-3 control-label") }} +
+ {{ asset_form.inflexible_device_sensor_ids(class_="form-control") }} + {% for error in asset_form.errors.inflexible_device_sensor_ids %} + [{{error}}] + {% endfor %} +
+
+ {% endif %}
{{ asset_form.attributes.label(class="col-sm-6 control-label") }}
diff --git a/flexmeasures/ui/tests/test_assets_forms.py b/flexmeasures/ui/tests/test_assets_forms.py new file mode 100644 index 000000000..ff2e5f834 --- /dev/null +++ b/flexmeasures/ui/tests/test_assets_forms.py @@ -0,0 +1,160 @@ +import pytest +from unittest.mock import patch + +from flexmeasures.ui.crud.assets import AssetForm +from flexmeasures.ui.tests.test_utils import NewAsset + + +@pytest.mark.parametrize( + "new_asset, allowed_price_sensor_data, expect_choices, expect_default", + [ + # No asset, no allowed_price_sensor_data + (None, dict(), [(-1, "--Select sensor id--")], (-1, "--Select sensor id--")), + # Have production_price_sensor_id, no allowed_price_sensor_data + ( + { + "set_production_price_sensor_id": True, + "asset_name": "Asset name", + "sensor_name": "Sensor name", + }, + dict(), + [(-1, "--Select sensor id--"), ("SENSOR_ID", "Asset name:Sensor name")], + ("SENSOR_ID", "Asset name:Sensor name"), + ), + # Have production_price_sensor_id and allowed_price_sensor_data + ( + { + "set_production_price_sensor_id": True, + "asset_name": "Asset name", + "sensor_name": "Sensor name", + }, + {1000: "Some asset name:Some sensor name"}, + [ + (-1, "--Select sensor id--"), + (1000, "Some asset name:Some sensor name"), + ("SENSOR_ID", "Asset name:Sensor name"), + ], + ("SENSOR_ID", "Asset name:Sensor name"), + ), + # No production_price_sensor_id, have allowed_price_sensor_data + ( + {"asset_name": "Asset name", "sensor_name": "Sensor name"}, + {1000: "Some asset name:Some sensor name"}, + [(-1, "--Select sensor id--"), (1000, "Some asset name:Some sensor name")], + (-1, "--Select sensor id--"), + ), + ], +) +def test_with_price_senors( + db, + setup_generic_asset_types, + setup_accounts, + new_asset, + allowed_price_sensor_data, + expect_choices, + expect_default, +): + form = AssetForm() + with NewAsset( + db, setup_generic_asset_types, setup_accounts, new_asset + ) as new_asset_decorator: + sensor_id = ( + new_asset_decorator.price_sensor.id + if new_asset_decorator and new_asset_decorator.price_sensor + else None + ) + expect_default = tuple( + sensor_id if element == "SENSOR_ID" else element + for element in expect_default + ) + expect_choices = [ + tuple( + sensor_id if element == "SENSOR_ID" else element for element in choice + ) + for choice in expect_choices + ] + + with patch( + "flexmeasures.ui.crud.assets.forms.get_allowed_price_sensor_data", + return_value=allowed_price_sensor_data, + ) as mock_method: + form.with_price_senors(new_asset_decorator.test_battery, 1) + assert mock_method.called_once_with(1) + # check production_price_sensor only as consumption_price is processed the same way + assert form.production_price_sensor_id.choices == expect_choices + assert form.production_price_sensor_id.default == expect_default + + +@pytest.mark.parametrize( + "new_asset, allowed_inflexible_sensor_data, expect_choices, expect_default", + [ + # No asset, no allowed_inflexible_sensor_data + (None, dict(), [(-1, "--Select sensor id--")], [-1]), + # Asset without linked sensors, no allowed_inflexible_sensor_data + ( + {"asset_name": "Asset name", "sensor_name": "Sensor name"}, + dict(), + [(-1, "--Select sensor id--")], + [-1], + ), + # Asset without linked sensors, have allowed_inflexible_sensor_data + ( + {"asset_name": "Asset name", "sensor_name": "Sensor name"}, + {1000: "Some asset name:Some sensor name"}, + [(-1, "--Select sensor id--"), (1000, "Some asset name:Some sensor name")], + [-1], + ), + # Have linked sensors, have allowed_inflexible_sensor_data + ( + { + "asset_name": "Asset name", + "sensor_name": "Sensor name", + "have_linked_sensors": True, + }, + {1000: "Some asset name:Some sensor name"}, + [ + (-1, "--Select sensor id--"), + (1000, "Some asset name:Some sensor name"), + ("SENSOR_ID", "Asset name:Sensor name"), + ], + ["SENSOR_ID"], + ), + ], +) +def test_with_inflexible_sensors( + db, + setup_generic_asset_types, + setup_accounts, + new_asset, + allowed_inflexible_sensor_data, + expect_choices, + expect_default, +): + form = AssetForm() + with NewAsset( + db, setup_generic_asset_types, setup_accounts, new_asset + ) as new_asset_decorator: + sensor_id = ( + new_asset_decorator.price_sensor.id + if new_asset_decorator and new_asset_decorator.price_sensor + else None + ) + expect_default = list( + sensor_id if element == "SENSOR_ID" else element + for element in expect_default + ) + expect_choices = [ + tuple( + sensor_id if element == "SENSOR_ID" else element for element in choice + ) + for choice in expect_choices + ] + + with patch( + "flexmeasures.ui.crud.assets.forms.get_allowed_inflexible_sensor_data", + return_value=allowed_inflexible_sensor_data, + ) as mock_method: + form.with_inflexible_sensors(new_asset_decorator.test_battery, 1) + assert mock_method.called_once_with(1) + assert form.inflexible_device_sensor_ids.choices == expect_choices + assert form.inflexible_device_sensor_ids.default == expect_default diff --git a/flexmeasures/ui/tests/test_assets_utils.py b/flexmeasures/ui/tests/test_assets_utils.py new file mode 100644 index 000000000..3a22a608d --- /dev/null +++ b/flexmeasures/ui/tests/test_assets_utils.py @@ -0,0 +1,196 @@ +import pytest + +from flexmeasures.ui.crud.assets import ( + get_allowed_price_sensor_data, + get_allowed_inflexible_sensor_data, +) +from flexmeasures.ui.tests.test_utils import NewAsset + + +@pytest.mark.parametrize( + "new_asset, account_argument, expect_new_asset_sensor", + [ + # No asset, no account - get the default sensors + (None, None, False), + # No account on asset and function call - get new asset sensors + ( + { + "asset_name": "No account asset", + "sensor_name": "No account asset sensor", + }, + None, + True, + ), + # Asset with account, function call without - do not get new asset sensors + ( + { + "asset_name": "Have account asset", + "sensor_name": "Have account asset sensor", + "set_account": True, + }, + None, + False, + ), + # Asset without account, function call with - do not get new asset sensors + ( + { + "asset_name": "Have account asset", + "sensor_name": "Have account asset sensor", + }, + 1, + False, + ), + # Asset has the same account that function call - get new asset sensors + ( + { + "asset_name": "Have account asset", + "sensor_name": "Have account asset sensor", + "set_account": True, + }, + 1, + True, + ), + # Asset and function call have different accounts - do not get new asset sensors + ( + { + "asset_name": "Have account asset", + "sensor_name": "Have account asset sensor", + "set_account": True, + }, + 1000, + False, + ), + # New asset sensor has different unit - do not get new asset sensors + ( + { + "asset_name": "No account asset", + "sensor_name": "No account asset sensor", + "sensor_unit": "kWh", + }, + None, + False, + ), + ], +) +def test_get_allowed_price_sensor_data( + db, + setup_generic_asset_types, + setup_accounts, + new_asset, + account_argument, + expect_new_asset_sensor, +): + with NewAsset( + db, setup_generic_asset_types, setup_accounts, new_asset + ) as new_asset_decorator: + price_sensor_data = get_allowed_price_sensor_data(account_argument) + if expect_new_asset_sensor: + assert len(price_sensor_data) == 3 + assert ( + price_sensor_data[new_asset_decorator.price_sensor.id] + == f'{new_asset["asset_name"]}:{new_asset["sensor_name"]}' + ) + + # we are adding these sensors in assets without account by default + assert price_sensor_data[1] == "epex:epex_da" + assert price_sensor_data[2] == "epex:epex_da_production" + + +@pytest.mark.parametrize( + "new_asset, account_argument, expect_new_asset_sensor", + [ + # No asset, no account - get the default sensors + (None, None, False), + # No account on asset and function call - get new asset sensors + ( + { + "asset_name": "No account asset", + "sensor_name": "No account asset sensor", + "sensor_unit": "kWh", + }, + None, + True, + ), + # Asset with account, function call without - do not get new asset sensors + ( + { + "asset_name": "Have account asset", + "sensor_name": "Have account asset sensor", + "sensor_unit": "kWh", + "set_account": True, + }, + None, + False, + ), + # Asset without account, function call with - do not get new asset sensors + ( + { + "asset_name": "Have account asset", + "sensor_name": "Have account asset sensor", + "sensor_unit": "kWh", + }, + 1, + False, + ), + # Asset has the same account that function call - get new asset sensors + ( + { + "asset_name": "Have account asset", + "sensor_name": "Have account asset sensor", + "sensor_unit": "kWh", + "set_account": True, + }, + 1, + True, + ), + # Asset and function call have different accounts - do not get new asset sensors + ( + { + "asset_name": "Have account asset", + "sensor_name": "Have account asset sensor", + "sensor_unit": "kWh", + "set_account": True, + }, + 1000, + False, + ), + # New asset sensor has energy unit - still use it + ( + { + "asset_name": "No account asset", + "sensor_name": "No account asset sensor", + "sensor_unit": "kW", + }, + None, + True, + ), + # New asset sensor has temperature unit - do not get new asset sensors + ( + { + "asset_name": "No account asset", + "sensor_name": "No account asset sensor", + "sensor_unit": "°C", + }, + None, + False, + ), + ], +) +def test_get_allowed_inflexible_sensor_data( + db, + setup_generic_asset_types, + setup_accounts, + new_asset, + account_argument, + expect_new_asset_sensor, +): + with NewAsset( + db, setup_generic_asset_types, setup_accounts, new_asset + ) as new_asset_decorator: + price_sensor_data = get_allowed_inflexible_sensor_data(account_argument) + if expect_new_asset_sensor: + assert len(price_sensor_data) == 1 + assert ( + price_sensor_data[new_asset_decorator.price_sensor.id] + == f'{new_asset["asset_name"]}:{new_asset["sensor_name"]}' + ) diff --git a/flexmeasures/ui/tests/test_utils.py b/flexmeasures/ui/tests/test_utils.py index a403a9d72..ae62bf6de 100644 --- a/flexmeasures/ui/tests/test_utils.py +++ b/flexmeasures/ui/tests/test_utils.py @@ -1,6 +1,14 @@ +from datetime import timedelta + from flexmeasures import Asset, AssetType, Account, Sensor +from flexmeasures.data.models.generic_assets import ( + GenericAsset, + GenericAssetInflexibleSensorRelationship, +) from flexmeasures.ui.utils.breadcrumb_utils import get_ancestry +from timely_beliefs.sensors.func_store.knowledge_horizons import x_days_ago_at_y_oclock + def test_get_ancestry(app, db): account = Account(name="Test Account") @@ -47,3 +55,73 @@ def test_get_ancestry(app, db): assert sensor_ancestry[-1]["type"] == "Sensor" assert sensor_ancestry[0]["type"] == "Account" assert all(b["type"] == "Asset" for b in sensor_ancestry[1:-1]) + + +class NewAsset: + def __init__(self, db, setup_generic_asset_types, setup_accounts, new_asset_data): + self.db = db + self.setup_generic_asset_types = setup_generic_asset_types + self.setup_accounts = setup_accounts + self.new_asset_data = new_asset_data + self.test_battery = None + self.price_sensor = None + + def __enter__(self): + if not self.new_asset_data: + return self + + owner = ( + self.setup_accounts["Prosumer"] + if self.new_asset_data.get("set_account") + else None + ) + self.test_battery = GenericAsset( + name=self.new_asset_data["asset_name"], + generic_asset_type=self.setup_generic_asset_types["battery"], + owner=owner, + attributes={"some-attribute": "some-value", "sensors_to_show": [1, 2]}, + ) + self.db.session.add(self.test_battery) + self.db.session.flush() + + unit = self.new_asset_data.get("sensor_unit", "EUR/MWh") + self.price_sensor = Sensor( + name=self.new_asset_data["sensor_name"], + generic_asset=self.test_battery, + event_resolution=timedelta(hours=1), + unit=unit, + knowledge_horizon=( + x_days_ago_at_y_oclock, + {"x": 1, "y": 12, "z": "Europe/Paris"}, + ), + attributes=dict( + daily_seasonality=True, + weekly_seasonality=True, + yearly_seasonality=True, + ), + ) + self.db.session.add(self.price_sensor) + self.db.session.flush() + + if self.new_asset_data.get("set_production_price_sensor_id"): + self.test_battery.production_price_sensor_id = self.price_sensor.id + self.db.session.add(self.test_battery) + if self.new_asset_data.get("have_linked_sensors"): + relationship = GenericAssetInflexibleSensorRelationship( + generic_asset_id=self.test_battery.id, + inflexible_sensor_id=self.price_sensor.id, + ) + self.db.session.add(relationship) + + self.db.session.commit() + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.new_asset_data: + return + + # Delete price_sensor and test_battery + self.db.session.delete(self.price_sensor) + self.db.session.delete(self.test_battery) + self.db.session.commit() diff --git a/setup.cfg b/setup.cfg index 82d4bc219..50aa25de0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,6 +11,7 @@ ignore = E501, W503, E203 per-file-ignores = flexmeasures/__init__.py:F401 flexmeasures/data/schemas/__init__.py:F401 + flexmeasures/ui/crud/assets/__init__.py:F401 [tool:pytest] addopts = --strict-markers