From d38cb688708066e5b8dfe607f18abbab568de5b9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 13 Jul 2023 13:14:38 +0200 Subject: [PATCH 1/8] fix: broken import after merge Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/schemas/__init__.py b/flexmeasures/data/schemas/__init__.py index 1317d3ab2..dbbf17f42 100644 --- a/flexmeasures/data/schemas/__init__.py +++ b/flexmeasures/data/schemas/__init__.py @@ -2,7 +2,6 @@ Data schemas (Marshmallow) """ -from .assets import LatitudeField, LongitudeField # noqa F401 from .generic_assets import GenericAssetIdField as AssetIdField # noqa F401 from .locations import LatitudeField, LongitudeField # noqa F401 from .sensors import SensorIdField # noqa F401 From 37e297204ec4ac1f220e0fd1c1beb315eea3c4b5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 13 Jul 2023 13:52:26 +0200 Subject: [PATCH 2/8] sunset: fm0 scheme Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensors.py | 21 ++---------- .../api/common/schemas/tests/test_sensors.py | 33 +++++-------------- 2 files changed, 11 insertions(+), 43 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensors.py b/flexmeasures/api/common/schemas/sensors.py index 3b4daa65e..6cae945fa 100644 --- a/flexmeasures/api/common/schemas/sensors.py +++ b/flexmeasures/api/common/schemas/sensors.py @@ -2,9 +2,6 @@ from marshmallow import fields from flexmeasures.api import FMValidationError -from flexmeasures.api.common.utils.api_utils import ( - get_sensor_by_generic_asset_type_and_location, -) from flexmeasures.utils.entity_address_utils import ( parse_entity_address, EntityAddressException, @@ -33,7 +30,8 @@ def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int: class SensorField(fields.Str): """Field that de-serializes to a Sensor, - and serializes a Sensor, Asset, Market or WeatherSensor into an entity address (string).""" + and serializes a Sensor, Asset, Market or WeatherSensor into an entity address (string). + """ # todo: when Actuators also get an entity address, refactor this class to EntityField, # where an Entity represents anything with an entity address: we currently foresee Sensors and Actuators @@ -58,20 +56,7 @@ def _deserialize(self, value, attr, obj, **kwargs) -> Sensor: try: ea = parse_entity_address(value, self.entity_type, self.fm_scheme) if self.fm_scheme == "fm0": - if self.entity_type == "connection": - sensor = Sensor.query.filter( - Sensor.id == ea["asset_id"] - ).one_or_none() - elif self.entity_type == "market": - sensor = Sensor.query.filter( - Sensor.name == ea["market_name"] - ).one_or_none() - elif self.entity_type == "weather_sensor": - sensor = get_sensor_by_generic_asset_type_and_location( - ea["weather_sensor_type_name"], ea["latitude"], ea["longitude"] - ) - else: - return NotImplemented + raise EntityAddressException("The fm0 scheme is no longer supported.") else: sensor = Sensor.query.filter(Sensor.id == ea["sensor_id"]).one_or_none() if sensor is not None: diff --git a/flexmeasures/api/common/schemas/tests/test_sensors.py b/flexmeasures/api/common/schemas/tests/test_sensors.py index 598d6221a..279109acd 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensors.py +++ b/flexmeasures/api/common/schemas/tests/test_sensors.py @@ -16,22 +16,6 @@ "fm1", "height", ), - ( - build_entity_address( - dict(market_name="epex_da"), "market", fm_scheme="fm0" - ), - "market", - "fm0", - "epex_da", - ), - ( - build_entity_address( - dict(owner_id=1, asset_id=4), "connection", fm_scheme="fm0" - ), - "connection", - "fm0", - "Test battery with no known prices", - ), ], ) def test_sensor_field_straightforward( @@ -47,9 +31,6 @@ def test_sensor_field_straightforward( sf = SensorField(entity_type, fm_scheme) deser = sf.deserialize(entity_address, None, None) assert deser.name == exp_deserialization_name - if fm_scheme == "fm0" and entity_type in ("connection", "market", "weather_sensor"): - # These entity types are deserialized to Sensors, which have no entity address under the fm0 scheme - return assert sf.serialize(entity_type, {entity_type: deser}) == entity_address @@ -57,10 +38,12 @@ def test_sensor_field_straightforward( "entity_address, entity_type, fm_scheme, error_msg", [ ( - "ea1.2021-01.io.flexmeasures:some.weird:identifier%that^is*not)used", + build_entity_address( + dict(market_name="epex_da"), "market", fm_scheme="fm0" + ), "market", "fm0", - "Could not parse", + "fm0 scheme is no longer supported", ), ( "ea1.2021-01.io.flexmeasures:fm1.some.weird:identifier%that^is*not)used", @@ -70,15 +53,15 @@ def test_sensor_field_straightforward( ), ( build_entity_address( - dict(market_name="non_existing_market"), "market", fm_scheme="fm0" + dict(sensor_id=99999999999999), "sensor", fm_scheme="fm1" ), - "market", - "fm0", + "sensor", + "fm1", "doesn't exist", ), ( build_entity_address(dict(sensor_id=-1), "sensor", fm_scheme="fm1"), - "market", + "sensor", "fm1", "Could not parse", ), From dd717d8c6ddd1272d50f6feccc056e9cbde377db Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 13 Jul 2023 13:57:42 +0200 Subject: [PATCH 3/8] docs: sunset fm0 scheme Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 4 ++++ documentation/api/notation.rst | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index b19fdf993..6fe80a950 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -5,6 +5,10 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. +v3.0-11 | 2023-07-13 + +- Removed undocumented compatibility of API v3 with the fm0 scheme for entity addresses. + v3.0-10 | 2023-06-12 """""""""""""""""""" diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index 6cf53bf47..3b5c6480d 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -94,7 +94,7 @@ It uses the fact that all FlexMeasures sensors have unique IDs. The ``fm0`` scheme is the original scheme. It identified different types of sensors (such as grid connections, weather sensors and markets) in different ways. -The ``fm0`` scheme has been deprecated and is no longer supported officially. +The ``fm0`` scheme has been sunset. Timeseries From 5fb02c34ad888283635be6ace5c0d553b42db634 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 13 Jul 2023 14:15:16 +0200 Subject: [PATCH 4/8] erratum: fm0 was not compatible with API v3 Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 4 ---- documentation/api/notation.rst | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 6fe80a950..b19fdf993 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -5,10 +5,6 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace. -v3.0-11 | 2023-07-13 - -- Removed undocumented compatibility of API v3 with the fm0 scheme for entity addresses. - v3.0-10 | 2023-06-12 """""""""""""""""""" diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index 3b5c6480d..1eafcc227 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -94,7 +94,7 @@ It uses the fact that all FlexMeasures sensors have unique IDs. The ``fm0`` scheme is the original scheme. It identified different types of sensors (such as grid connections, weather sensors and markets) in different ways. -The ``fm0`` scheme has been sunset. +The ``fm0`` scheme has been sunset since API version 3. Timeseries From a1ceaf6cd1a6d2bc87d2ca92a45203d4b68f4664 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 13 Jul 2023 14:16:42 +0200 Subject: [PATCH 5/8] delete: obsolete API utils Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensors.py | 2 +- flexmeasures/api/common/utils/api_utils.py | 58 -- flexmeasures/api/common/utils/validators.py | 779 +------------------- 3 files changed, 2 insertions(+), 837 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensors.py b/flexmeasures/api/common/schemas/sensors.py index 6cae945fa..79b82964d 100644 --- a/flexmeasures/api/common/schemas/sensors.py +++ b/flexmeasures/api/common/schemas/sensors.py @@ -30,7 +30,7 @@ def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int: class SensorField(fields.Str): """Field that de-serializes to a Sensor, - and serializes a Sensor, Asset, Market or WeatherSensor into an entity address (string). + and serializes a Sensor into an entity address (string). """ # todo: when Actuators also get an entity address, refactor this class to EntityField, diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 302f1d89f..3da581904 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -14,13 +14,9 @@ import timely_beliefs as tb from flexmeasures.data import db -from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType -from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.models.weather import WeatherSensor from flexmeasures.data.utils import save_to_db as modern_save_to_db from flexmeasures.api.common.responses import ( invalid_replacement, - unrecognized_sensor, ResponseTuple, request_processed, already_received_and_successfully_processed, @@ -240,60 +236,6 @@ def unique_ever_seen(iterable: Sequence, selector: Sequence): return u, s -def get_sensor_by_generic_asset_type_and_location( - generic_asset_type_name: str, latitude: float = 0, longitude: float = 0 -) -> Union[Sensor, ResponseTuple]: - """ - Search a sensor by generic asset type and location. - Can create a sensor if needed (depends on API mode) - and then inform the requesting user which one to use. - """ - # Look for the Sensor object - sensor = ( - Sensor.query.join(GenericAsset) - .join(GenericAssetType) - .filter(GenericAssetType.name == generic_asset_type_name) - .filter(GenericAsset.generic_asset_type_id == GenericAssetType.id) - .filter(GenericAsset.latitude == latitude) - .filter(GenericAsset.longitude == longitude) - .filter(Sensor.generic_asset_id == GenericAsset.id) - .one_or_none() - ) - if sensor is None: - create_sensor_if_unknown = False - if current_app.config.get("FLEXMEASURES_MODE", "") == "play": - create_sensor_if_unknown = True - - # either create a new weather sensor and post to that - if create_sensor_if_unknown: - current_app.logger.info("CREATING NEW WEATHER SENSOR...") - weather_sensor = WeatherSensor( - name="Weather sensor for %s at latitude %s and longitude %s" - % (generic_asset_type_name, latitude, longitude), - weather_sensor_type_name=generic_asset_type_name, - latitude=latitude, - longitude=longitude, - ) - db.session.add(weather_sensor) - db.session.flush() # flush so that we can reference the new object in the current db session - sensor = weather_sensor.corresponding_sensor - - # or query and return the nearest sensor and let the requesting user post to that one - else: - nearest_weather_sensor = WeatherSensor.query.order_by( - WeatherSensor.great_circle_distance( - latitude=latitude, longitude=longitude - ).asc() - ).first() - if nearest_weather_sensor is not None: - return unrecognized_sensor( - *nearest_weather_sensor.location, - ) - else: - return unrecognized_sensor() - return sensor - - def enqueue_forecasting_jobs( forecasting_jobs: list[Job] | None = None, ): diff --git a/flexmeasures/api/common/utils/validators.py b/flexmeasures/api/common/utils/validators.py index 50ce9f22c..80f2e1081 100644 --- a/flexmeasures/api/common/utils/validators.py +++ b/flexmeasures/api/common/utils/validators.py @@ -2,22 +2,17 @@ from datetime import datetime, timedelta from functools import wraps -from typing import List, Tuple, Union, Optional +from typing import Tuple, Union, Optional import re import isodate from isodate.isoerror import ISO8601Error -import inflect -from inflection import pluralize -from pandas.tseries.frequencies import to_offset from flask import request, current_app from flask_json import as_json -from flask_security import current_user import marshmallow from webargs.flaskparser import parser -from flexmeasures.api.common.schemas.sensors import SensorField from flexmeasures.data.schemas.times import DurationField from flexmeasures.api.common.responses import ( # noqa: F401 required_info_missing, @@ -36,16 +31,6 @@ unrecognized_connection_group, unrecognized_asset, ) -from flexmeasures.api.common.utils.api_utils import ( - get_form_from_request, - parse_as_list, - contains_empty_items, - upsample_values, -) -from flexmeasures.data import db -from flexmeasures.data.models.data_sources import DataSource -from flexmeasures.data.services.users import get_users -from flexmeasures.utils.time_utils import server_now """ This module has validators used by API endpoints <= 2.0 to describe @@ -56,53 +41,6 @@ """ -p = inflect.engine() - - -def validate_user_sources(sources: Union[int, str, List[Union[int, str]]]) -> List[int]: - """ - Return a list of user-based data source ids, given: - - one or more user ids - - one or more account role names - """ - sources = ( - sources if isinstance(sources, list) else [sources] - ) # Make sure sources is a list - user_source_ids: List[int] = [] - for source in sources: - if isinstance(source, int): # Parse as user id - try: - user_source_ids.extend( - db.session.query(DataSource.id) - .filter(DataSource.user_id == source) - .one_or_none() - ) - except TypeError: - current_app.logger.warning("Could not retrieve data source %s" % source) - pass - else: # Parse as account role name - user_ids = [user.id for user in get_users(account_role_name=source)] - user_source_ids.extend( - [ - params[0] - for params in db.session.query(DataSource.id) - .filter(DataSource.user_id.in_(user_ids)) - .all() - ] - ) - return list(set(user_source_ids)) # only unique ids - - -def include_current_user_source_id(source_ids: List[int]) -> List[int]: - """Includes the source id of the current user.""" - source_ids.extend( - db.session.query(DataSource.id) - .filter(DataSource.user_id == current_user.id) - .one_or_none() - ) - return list(set(source_ids)) # only unique source ids - - def parse_horizon(horizon_str: str) -> Tuple[Optional[timedelta], bool]: """ Validates whether a horizon string represents a valid ISO 8601 (repeating) time interval. @@ -159,32 +97,6 @@ def parse_duration( return None -def parse_isodate_str(start: str) -> Union[datetime, None]: - """ - Validates whether the string 'start' is a valid ISO 8601 datetime. - """ - try: - return isodate.parse_datetime(start) - except (ISO8601Error, AttributeError): - return None - - -def valid_sensor_units(sensor: str) -> List[str]: - """ - Returns the accepted units for this sensor. - """ - if sensor == "temperature": - return ["°C", "0C"] - elif sensor == "irradiance": - return ["kW/m²", "kW/m2"] - elif sensor == "wind speed": - return ["m/s"] - else: - raise NotImplementedError( - "Unknown sensor or physical unit, cannot determine valid units." - ) - - def optional_duration_accepted(default_duration: timedelta): """Decorator which specifies that a GET or POST request accepts an optional duration. It parses relevant form data and sets the "duration" keyword param. @@ -227,692 +139,3 @@ def decorated_service(*args, **kwargs): return decorated_service return wrapper - - -def optional_user_sources_accepted( - default_source: int | str | list[int | str] | None = None, -): - """Decorator which specifies that a GET or POST request accepts an optional source or list of data sources. - It parses relevant form data and sets the "user_source_ids" keyword parameter. - - Data originating from the requesting user is included by default. - That is, user_source_ids always includes the source id of the requesting user. - - Each source should either be a known USEF role name or a user id. - We'll parse them as a list of source ids. - - Case 1: - If a request states one or more data sources, then we'll only query those, in addition to the user's own data. - Default sources specified in the decorator (see example below) are ignored. - - Case 2: - If a request does not state any data sources, a list of default sources will be used. - - Case 2A: - Default sources can be specified in the decorator (see example below). - - Case 2B: - If no default sources are specified in the decorator, all sources are included. - - Example: - - @app.route('/getMeterData') - @optional_sources_accepted("MDC") - def get_meter_data(user_source_ids): - return 'Meter data posted' - - The source ids then include the user's own id, - and ids of other users that are registered as a Meter Data Company. - - If the message specifies: - - .. code-block:: json - - { - "sources": ["Prosumer", "ESCo"] - } - - The source ids then include the user's own id, - and ids of other users whose organisation account is registered as a Prosumer and/or Energy Service Company. - """ - - def wrapper(fn): - @wraps(fn) - @as_json - def decorated_service(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking 'source' from request." - ) - return invalid_method(request.method) - - if "source" in form: - validated_user_source_ids = validate_user_sources(form["source"]) - if None in validated_user_source_ids: - return invalid_source(form["source"]) - kwargs["user_source_ids"] = include_current_user_source_id( - validated_user_source_ids - ) - elif default_source is not None: - kwargs["user_source_ids"] = include_current_user_source_id( - validate_user_sources(default_source) - ) - else: - kwargs["user_source_ids"] = None - - return fn(*args, **kwargs) - - return decorated_service - - return wrapper - - -def optional_prior_accepted( - ex_post: bool = False, infer_missing: bool = True, infer_missing_play: bool = False -): - """Decorator which specifies that a GET or POST request accepts an optional prior. - It parses relevant form data and sets the "prior" keyword param. - - Interpretation for GET requests: - - Denotes "at least before " - - This results in the filter belief_time_window = (None, prior) - - Interpretation for POST requests: - - Denotes "recorded to some datetime, - - this results in the assignment belief_time = prior - - :param ex_post: if True, only ex-post datetimes are allowed. - :param infer_missing: if True, servers assume that the belief_time of posted - values is server time. This setting is meant to be used for POST requests. - :param infer_missing_play: if True, servers in play mode assume that the belief_time of posted - values is server time. This setting is meant to be used for POST requests. - """ - - def wrapper(fn): - @wraps(fn) - @as_json - def decorated_service(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking 'prior' from request." - ) - return invalid_method(request.method) - - if "prior" in form: - prior = parse_isodate_str(form["prior"]) - if ex_post is True: - start = parse_isodate_str(form["start"]) - duration = parse_duration(form["duration"], start) - # todo: validate start and duration (refactor already duplicate code from period_required and optional_horizon_accepted) - knowledge_time = ( - start + duration - ) # todo: take into account knowledge horizon function - if prior < knowledge_time: - extra_info = "Meter data can only be observed after the fact." - return invalid_horizon(extra_info) - elif infer_missing is True or ( - infer_missing_play is True - and current_app.config.get("FLEXMEASURES_MODE", "") == "play" - ): - # A missing prior is inferred by the server - prior = server_now() - else: - # Otherwise, a missing prior is fine (a horizon may still be inferred by the server) - prior = None - - kwargs["prior"] = prior - return fn(*args, **kwargs) - - return decorated_service - - return wrapper - - -def optional_horizon_accepted( # noqa C901 - ex_post: bool = False, - infer_missing: bool = True, - infer_missing_play: bool = False, - accept_repeating_interval: bool = False, -): - """Decorator which specifies that a GET or POST request accepts an optional horizon. - The horizon should be in accordance with the ISO 8601 standard. - It parses relevant form data and sets the "horizon" keyword param (a timedelta). - - Interpretation for GET requests: - - Denotes "at least before the fact (positive horizon), - or at most after the fact (negative horizon)" - - This results in the filter belief_horizon_window = (horizon, None) - - Interpretation for POST requests: - - Denotes "at before the fact (positive horizon), - or at after the fact (negative horizon)" - - this results in the assignment belief_horizon = horizon - - For example: - - @app.route('/postMeterData') - @optional_horizon_accepted() - def post_meter_data(horizon): - return 'Meter data posted' - - :param ex_post: if True, only non-positive horizons are allowed. - :param infer_missing: if True, servers assume that the belief_horizon of posted - values is 0 hours. This setting is meant to be used for POST requests. - :param infer_missing_play: if True, servers in play mode assume that the belief_horizon of posted - values is 0 hours. This setting is meant to be used for POST requests. - :param accept_repeating_interval: if True, the "rolling" keyword param is also set - (this was used for POST requests before v2.0) - """ - - def wrapper(fn): - @wraps(fn) - @as_json - def decorated_service(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking 'horizon' from request." - ) - return invalid_method(request.method) - - rolling = True - if "horizon" in form: - horizon, rolling = parse_horizon(form["horizon"]) - if horizon is None: - current_app.logger.warning("Cannot parse 'horizon' value") - return invalid_horizon() - elif ex_post is True: - if horizon > timedelta(hours=0): - extra_info = "Meter data must have a zero or negative horizon to indicate observations after the fact." - return invalid_horizon(extra_info) - elif rolling is True and accept_repeating_interval is False: - extra_info = ( - "API versions 2.0 and higher use regular ISO 8601 durations instead of repeating time intervals. " - "For example: R/P1D should be replaced by P1D." - ) - return invalid_horizon(extra_info) - elif infer_missing is True or ( - infer_missing_play is True - and current_app.config.get("FLEXMEASURES_MODE", "") == "play" - ): - # A missing horizon is set to zero - horizon = timedelta(hours=0) - else: - # Otherwise, a missing horizon is fine (a prior may still be inferred by the server) - horizon = None - - kwargs["horizon"] = horizon - if infer_missing is True and accept_repeating_interval is True: - kwargs["rolling"] = rolling - return fn(*args, **kwargs) - - return decorated_service - - return wrapper - - -def unit_required(fn): - """Decorator which specifies that a GET or POST request must specify a unit. - It parses relevant form data and sets the "unit keyword param. - Example: - - @app.route('/postMeterData') - @unit_required - def post_meter_data(unit): - return 'Meter data posted' - - The message must specify a 'unit'. - """ - - @wraps(fn) - @as_json - def wrapper(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking 'unit' from request." - ) - return invalid_method(request.method) - - if "unit" in form: - unit = form["unit"] - else: - current_app.logger.warning("Request missing 'unit'.") - return invalid_unit(quantity=None, units=None) - - kwargs["unit"] = unit - return fn(*args, **kwargs) - - return wrapper - - -def period_required(fn): - """Decorator which specifies that a GET or POST request must specify a time period (by start and duration). - It parses relevant form data and sets the "start" and "duration" keyword params. - Example: - - @app.route('/postMeterData') - @period_required - def post_meter_data(period): - return 'Meter data posted' - - The message must specify a 'start' and a 'duration' in accordance with the ISO 8601 standard. - This decorator should not be used together with optional_duration_accepted. - """ - - @wraps(fn) - @as_json - def wrapper(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking 'start' and 'duration' from request." - ) - return invalid_method(request.method) - - if "start" in form: - start = parse_isodate_str(form["start"]) - if not start: - current_app.logger.warning("Cannot parse 'start' value") - return invalid_period() - if start.tzinfo is None: - current_app.logger.warning("Cannot parse timezone of 'start' value") - return invalid_timezone( - "Start time should explicitly state a timezone." - ) - else: - current_app.logger.warning("Request missing 'start'.") - return invalid_period() - kwargs["start"] = start - if "duration" in form: - duration = parse_duration(form["duration"], start) - if not duration: - current_app.logger.warning("Cannot parse 'duration' value") - return invalid_period() - else: - current_app.logger.warning("Request missing 'duration'.") - return invalid_period() - kwargs["duration"] = duration - return fn(*args, **kwargs) - - return wrapper - - -def assets_required( - generic_asset_type_name: str, plural_name: str | None = None, groups_name="groups" -): - """Decorator which specifies that a GET or POST request must specify one or more assets. - It parses relevant form data and sets the "generic_asset_name_groups" keyword param. - Example: - - @app.route('/postMeterData') - @assets_required("connection", plural_name="connections") - def post_meter_data(generic_asset_name_groups): - return 'Meter data posted' - - Given this example, the message must specify one or more assets as "connections". - If that is the case, then the assets are passed to the function as generic_asset_name_groups. - - Connections can be listed in one of the following ways: - - value of 'connection' key (for a single asset) - - values of 'connections' key (for multiple assets that have the same timeseries data) - - values of the 'connection' and/or 'connections' keys listed under the 'groups' key - (for multiple assets with different timeseries data) - """ - if plural_name is None: - plural_name = pluralize(generic_asset_type_name) - - def wrapper(fn): - @wraps(fn) - @as_json - def decorated_service(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking '%s' from request." - % plural_name - ) - return invalid_method(request.method) - - if generic_asset_type_name in form: - generic_asset_name_groups = [ - parse_as_list(form[generic_asset_type_name]) - ] - elif plural_name in form: - generic_asset_name_groups = [parse_as_list(form[plural_name])] - elif groups_name in form: - generic_asset_name_groups = [] - for group in form["groups"]: - if generic_asset_type_name in group: - generic_asset_name_groups.append( - parse_as_list(group[generic_asset_type_name]) - ) - elif plural_name in group: - generic_asset_name_groups.append( - parse_as_list(group[plural_name]) - ) - else: - current_app.logger.warning( - "Group %s missing %s" % (group, plural_name) - ) - return unrecognized_connection_group() - else: - current_app.logger.warning("Request missing %s or group." % plural_name) - return unrecognized_connection_group() - - if not contains_empty_items(generic_asset_name_groups): - kwargs["generic_asset_name_groups"] = generic_asset_name_groups - return fn(*args, **kwargs) - else: - current_app.logger.warning("Request includes empty %s." % plural_name) - return unrecognized_connection_group() - - return decorated_service - - return wrapper - - -def values_required(fn): - """Decorator which specifies that a GET or POST request must specify one or more values. - It parses relevant form data and sets the "value_groups" keyword param. - Example: - - @app.route('/postMeterData') - @values_required - def post_meter_data(value_groups): - return 'Meter data posted' - - The message must specify one or more values. If that is the case, then the values are passed to the - function as value_groups. - """ - - @wraps(fn) - @as_json - def wrapper(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking 'values' from request." - ) - return invalid_method(request.method) - - if "value" in form: - value_groups = [parse_as_list(form["value"], of_type=float)] - elif "values" in form: - value_groups = [parse_as_list(form["values"], of_type=float)] - elif "groups" in form: - value_groups = [] - for group in form["groups"]: - if "value" in group: - value_groups.append(parse_as_list(group["value"], of_type=float)) - elif "values" in group: - value_groups.append(parse_as_list(group["values"], of_type=float)) - else: - current_app.logger.warning("Group %s missing value(s)" % group) - return ptus_incomplete() - else: - current_app.logger.warning("Request missing value(s) or group.") - return ptus_incomplete() - - if not contains_empty_items(value_groups): - kwargs["value_groups"] = value_groups - return fn(*args, **kwargs) - else: - extra_info = "Request includes empty or ill-formatted value(s)." - current_app.logger.warning(extra_info) - return ptus_incomplete(extra_info) - - return wrapper - - -def type_accepted(message_type: str): - """Decorator which specifies that a GET or POST request must specify the specified message type. Example: - - @app.route('/postMeterData') - @type_accepted('PostMeterDataRequest') - def post_meter_data(): - return 'Meter data posted' - - The message must specify 'PostMeterDataRequest' as its 'type'. - - :param message_type: The message type. - """ - - def wrapper(fn): - @wraps(fn) - @as_json - def decorated_service(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking 'type' from request." - ) - return invalid_method(request.method) - elif "type" not in form: - current_app.logger.warning("Request is missing message type.") - return no_message_type() - elif form["type"] != message_type: - current_app.logger.warning("Type is not accepted for this endpoint.") - return invalid_message_type(message_type) - else: - return fn(*args, **kwargs) - - return decorated_service - - return wrapper - - -def units_accepted(quantity: str, *units: str): - """Decorator which specifies that a GET or POST request must specify one of the - specified physical units. First parameter specifies the physical or economical quantity. - It parses relevant form data and sets the "unit" keyword param. - Example: - - @app.route('/postMeterData') - @units_accepted("power", 'MW', 'MWh') - def post_meter_data(unit): - return 'Meter data posted' - - The message must either specify 'MW' or 'MWh' as the unit. - - :param quantity: The physical or economic quantity - :param units: The possible units. - """ - - def wrapper(fn): - @wraps(fn) - @as_json - def decorated_service(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking 'unit' from request." - ) - return invalid_method(request.method) - elif "unit" not in form: - current_app.logger.warning("Request is missing unit.") - return invalid_unit(quantity, units) - elif form["unit"] not in units: - current_app.logger.warning( - "Unit %s is not accepted as one of %s." % (form["unit"], units) - ) - return invalid_unit(quantity, units) - else: - kwargs["unit"] = form["unit"] - return fn(*args, **kwargs) - - return decorated_service - - return wrapper - - -def post_data_checked_for_required_resolution( - entity_type: str, fm_scheme: str -): # noqa: C901 - """Decorator which checks that a POST request receives time series data with the event resolutions - required by the sensor. It sets the "resolution" keyword argument. - If the resolution in the data is a multiple of the sensor resolution, values are upsampled to the sensor resolution. - Finally, this decorator also checks if all sensors have the same event_resolution and complains otherwise. - - The resolution of the data is inferred from the duration and the number of values. - Therefore, the decorator should follow after the values_required, period_required and assets_required decorators. - Example: - - @app.route('/postMeterData') - @values_required - @period_required - @assets_required("connection") - @post_data_checked_for_required_resolution("connection") - def post_meter_data(value_groups, start, duration, generic_asset_name_groups, resolution) - return 'Meter data posted' - """ - - def wrapper(fn): - @wraps(fn) - @as_json - def decorated_service(*args, **kwargs): - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for inferring resolution from request." - ) - return invalid_method(request.method) - - if not all( - key in kwargs - for key in [ - "value_groups", - "start", - "duration", - ] - ): - current_app.logger.warning("Could not infer resolution.") - fields = ("values", "start", "duration") - return required_info_missing(fields, "Resolution cannot be inferred.") - if "generic_asset_name_groups" not in kwargs: - return required_info_missing( - (entity_type), - "Required resolution cannot be found without asset info.", - ) - - # Calculating (inferring) the resolution in the POSTed data - inferred_resolution = ( - (kwargs["start"] + kwargs["duration"]) - kwargs["start"] - ) / len(kwargs["value_groups"][0]) - - # Finding the required resolution for sensors affected in this request - required_resolution = None - last_sensor = None - for asset_group in kwargs["generic_asset_name_groups"]: - for asset_descriptor in asset_group: - # Getting the sensor - sensor = SensorField(entity_type, fm_scheme).deserialize( - asset_descriptor - ) - if sensor is None: - return unrecognized_asset( - f"Failed to look up asset by {asset_descriptor}" - ) - # Complain if sensors don't all require the same resolution - if ( - required_resolution is not None - and sensor.event_resolution != required_resolution - ): - return conflicting_resolutions( - f"Cannot send data for both {sensor} and {last_sensor}." - ) - # Setting the resolution & remembering last looked-at sensor - required_resolution = sensor.event_resolution - last_sensor = sensor - - # if inferred resolution is a multiple from required_solution, we can upsample_values - # todo: next line fails on sensors with 0 resolution - if inferred_resolution % required_resolution == timedelta(hours=0): - for i in range(len(kwargs["value_groups"])): - kwargs["value_groups"][i] = upsample_values( - kwargs["value_groups"][i], - from_resolution=inferred_resolution, - to_resolution=required_resolution, - ) - inferred_resolution = required_resolution - - if inferred_resolution != required_resolution: - current_app.logger.warning( - f"Resolution {inferred_resolution} is not accepted. We require {required_resolution}." - ) - return unapplicable_resolution( - isodate.duration_isoformat(required_resolution) - ) - else: - kwargs["resolution"] = inferred_resolution - return fn(*args, **kwargs) - - return decorated_service - - return wrapper - - -def get_data_downsampling_allowed(entity_type: str, fm_scheme: str): - """Decorator which allows downsampling of data which a GET request returns. - It checks for a form parameter "resolution". - If that is given and is a multiple of the sensor's event_resolution, - downsampling is performed on the data. This is done by setting the "resolution" - keyword parameter, which is obeyed by collect_time_series_data and used - in resampling. - - The original resolution of the data is the event_resolution of the sensor. - Therefore, the decorator should follow after the assets_required decorator. - - Example: - - @app.route('/getMeterData') - @assets_required("connection") - @get_data_downsampling_allowed("connection") - def get_meter_data(generic_asset_name_groups, resolution): - return data - - """ - - def wrapper(fn): - @wraps(fn) - @as_json - def decorated_service(*args, **kwargs): - kwargs[ - "resolution" - ] = None # using this decorator means you can expect this attribute, None means default - form = get_form_from_request(request) - if form is None: - current_app.logger.warning( - "Unsupported request method for unpacking 'resolution' from request." - ) - return invalid_method(request.method) - - if "resolution" in form and form["resolution"]: - ds_resolution = parse_duration(form["resolution"]) - if ds_resolution is None: - return invalid_resolution_str(form["resolution"]) - # Check if the resolution can be applied to all sensors (if it is a multiple - # of the event_resolution(s) and thus downsampling is possible) - for asset_group in kwargs["generic_asset_name_groups"]: - for asset_descriptor in asset_group: - sensor = SensorField(entity_type, fm_scheme).deserialize( - asset_descriptor - ) - if sensor is None: - return unrecognized_asset() - sensor_resolution = sensor.event_resolution - if ds_resolution % sensor_resolution != timedelta(minutes=0): - return unapplicable_resolution( - f"{isodate.duration_isoformat(sensor_resolution)} or a multiple hereof." - ) - kwargs["resolution"] = to_offset( - isodate.parse_duration(form["resolution"]) - ).freqstr # Convert ISO period string to pandas frequency string - - return fn(*args, **kwargs) - - return decorated_service - - return wrapper From d7af7d570366db402f46992de887d2b08f44f54e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 13 Jul 2023 14:31:23 +0200 Subject: [PATCH 6/8] update docstring and set default to fm1 scheme Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensors.py | 4 ++-- flexmeasures/api/common/schemas/tests/test_sensors.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensors.py b/flexmeasures/api/common/schemas/sensors.py index 79b82964d..95a4ec8c4 100644 --- a/flexmeasures/api/common/schemas/sensors.py +++ b/flexmeasures/api/common/schemas/sensors.py @@ -39,12 +39,12 @@ class SensorField(fields.Str): def __init__( self, entity_type: str, - fm_scheme: str, + fm_scheme: str = "fm1", *args, **kwargs, ): """ - :param entity_type: "sensor", "connection", "market" or "weather_sensor" + :param entity_type: "sensor" (in the future, possibly also another type of resource that is assigned an entity address) :param fm_scheme: "fm0" or "fm1" """ self.entity_type = entity_type diff --git a/flexmeasures/api/common/schemas/tests/test_sensors.py b/flexmeasures/api/common/schemas/tests/test_sensors.py index 279109acd..fe80811f7 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensors.py +++ b/flexmeasures/api/common/schemas/tests/test_sensors.py @@ -47,7 +47,7 @@ def test_sensor_field_straightforward( ), ( "ea1.2021-01.io.flexmeasures:fm1.some.weird:identifier%that^is*not)used", - "market", + "sensor", "fm1", "Could not parse", ), From ae84eebe52894572d4757af12ee5d8f5b39730d9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 13 Jul 2023 14:35:10 +0200 Subject: [PATCH 7/8] delete: remove obsolete WeatherSensor class Signed-off-by: F.N. Claessen --- .../models/forecasting/model_spec_factory.py | 5 +- flexmeasures/data/models/weather.py | 213 ------------------ 2 files changed, 2 insertions(+), 216 deletions(-) diff --git a/flexmeasures/data/models/forecasting/model_spec_factory.py b/flexmeasures/data/models/forecasting/model_spec_factory.py index abaaa0f70..190b524a9 100644 --- a/flexmeasures/data/models/forecasting/model_spec_factory.py +++ b/flexmeasures/data/models/forecasting/model_spec_factory.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional from datetime import datetime, timedelta, tzinfo from pprint import pformat import logging @@ -20,7 +20,6 @@ import pandas as pd from flexmeasures.data.models.time_series import Sensor, TimedBelief -from flexmeasures.data.models.weather import WeatherSensor from flexmeasures.data.models.forecasting.utils import ( create_lags, set_training_and_testing_dates, @@ -229,7 +228,7 @@ def _parameterise_forecasting_by_asset_and_asset_type( def get_normalization_transformation_from_sensor_attributes( - sensor: Union[Sensor, WeatherSensor], + sensor: Sensor, ) -> Optional[Transformation]: """ Transform data to be normal, using the BoxCox transformation. Lambda parameter is chosen diff --git a/flexmeasures/data/models/weather.py b/flexmeasures/data/models/weather.py index 09725c57d..0de4ddfe0 100644 --- a/flexmeasures/data/models/weather.py +++ b/flexmeasures/data/models/weather.py @@ -1,23 +1,7 @@ -from typing import Dict, Tuple - -import timely_beliefs as tb -from sqlalchemy.ext.hybrid import hybrid_method -from sqlalchemy.sql.expression import func -from sqlalchemy.schema import UniqueConstraint - from flexmeasures.data import db -from flexmeasures.data.models.legacy_migration_utils import ( - copy_old_sensor_attributes, - get_old_model_type, -) -from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.models.generic_assets import ( - create_generic_asset, - GenericAsset, GenericAssetType, ) -from flexmeasures.utils import geo_utils -from flexmeasures.utils.entity_address_utils import build_entity_address from flexmeasures.utils.flexmeasures_inflection import humanize @@ -44,200 +28,3 @@ def __init__(self, **kwargs): def __repr__(self): return "" % self.name - - -class WeatherSensor(db.Model, tb.SensorDBMixin): - """ - A weather sensor has a location on Earth and measures weather values of a certain weather sensor type, such as - temperature, wind speed and irradiance. - - This model is now considered legacy. See GenericAsset and Sensor. - """ - - id = db.Column( - db.Integer, db.ForeignKey("sensor.id"), primary_key=True, autoincrement=True - ) - name = db.Column(db.String(80), unique=True) - display_name = db.Column(db.String(80), default="", unique=False) - weather_sensor_type_name = db.Column( - db.String(80), db.ForeignKey("weather_sensor_type.name"), nullable=False - ) - # latitude is the North/South coordinate - latitude = db.Column(db.Float, nullable=False) - # longitude is the East/West coordinate - longitude = db.Column(db.Float, nullable=False) - - # only one sensor of any type is needed at one location - __table_args__ = ( - UniqueConstraint( - "weather_sensor_type_name", - "latitude", - "longitude", - name="weather_sensor_type_name_latitude_longitude_key", - ), - ) - - def __init__(self, **kwargs): - - super(WeatherSensor, self).__init__(**kwargs) - - # Create a new Sensor with unique id across assets, markets and weather sensors - if "id" not in kwargs: - - weather_sensor_type = get_old_model_type( - kwargs, - WeatherSensorType, - "weather_sensor_type_name", - "sensor_type", # NB not "weather_sensor_type" (slight inconsistency in this old sensor class) - ) - - generic_asset_kwargs = { - **kwargs, - **copy_old_sensor_attributes( - self, - old_sensor_type_attributes=[], - old_sensor_attributes=[ - "display_name", - ], - old_sensor_type=weather_sensor_type, - ), - } - new_generic_asset = create_generic_asset( - "weather_sensor", **generic_asset_kwargs - ) - new_sensor = Sensor( - name=kwargs["name"], - generic_asset=new_generic_asset, - **copy_old_sensor_attributes( - self, - old_sensor_type_attributes=[ - "daily_seasonality", - "weekly_seasonality", - "yearly_seasonality", - ], - old_sensor_attributes=[ - "display_name", - ], - old_sensor_type=weather_sensor_type, - ), - ) - db.session.add(new_sensor) - db.session.flush() # generates the pkey for new_sensor - new_sensor_id = new_sensor.id - else: - # The UI may initialize WeatherSensor objects from API form data with a known id - new_sensor_id = kwargs["id"] - - self.id = new_sensor_id - - # Copy over additional columns from (newly created) WeatherSensor to (newly created) Sensor - if "id" not in kwargs: - db.session.add(self) - db.session.flush() # make sure to generate each column for the old sensor - new_sensor.unit = self.unit - new_sensor.event_resolution = self.event_resolution - new_sensor.knowledge_horizon_fnc = self.knowledge_horizon_fnc - new_sensor.knowledge_horizon_par = self.knowledge_horizon_par - - @property - def entity_address_fm0(self) -> str: - """Entity address under the fm0 scheme for entity addresses.""" - return build_entity_address( - dict( - weather_sensor_type_name=self.weather_sensor_type_name, - latitude=self.latitude, - longitude=self.longitude, - ), - "weather_sensor", - fm_scheme="fm0", - ) - - @property - def entity_address(self) -> str: - """Entity address under the latest fm scheme for entity addresses.""" - return build_entity_address( - dict(sensor_id=self.id), - "sensor", - ) - - @property - def corresponding_sensor(self) -> Sensor: - return db.session.query(Sensor).get(self.id) - - @property - def generic_asset(self) -> GenericAsset: - return db.session.query(GenericAsset).get(self.corresponding_sensor.id) - - def get_attribute(self, attribute: str): - """Looks for the attribute on the corresponding Sensor. - - This should be used by all code to read these attributes, - over accessing them directly on this class, - as this table is in the process to be replaced by the Sensor table. - """ - return self.corresponding_sensor.get_attribute(attribute) - - @property - def weather_unit(self) -> float: - """Return the 'unit' property of the generic asset, just with a more insightful name.""" - return self.unit - - @property - def location(self) -> Tuple[float, float]: - return self.latitude, self.longitude - - @hybrid_method - def great_circle_distance(self, **kwargs): - """Query great circle distance (in km). - - Can be called with an object that has latitude and longitude properties, for example: - - great_circle_distance(object=asset) - - Can also be called with latitude and longitude parameters, for example: - - great_circle_distance(latitude=32, longitude=54) - great_circle_distance(lat=32, lng=54) - - """ - other_location = geo_utils.parse_lat_lng(kwargs) - if None in other_location: - return None - return geo_utils.earth_distance(self.location, other_location) - - @great_circle_distance.expression - def great_circle_distance(self, **kwargs): - """Query great circle distance (unclear if in km or in miles). - - Can be called with an object that has latitude and longitude properties, for example: - - great_circle_distance(object=asset) - - Can also be called with latitude and longitude parameters, for example: - - great_circle_distance(latitude=32, longitude=54) - great_circle_distance(lat=32, lng=54) - - """ - other_location = geo_utils.parse_lat_lng(kwargs) - if None in other_location: - return None - return func.earth_distance( - func.ll_to_earth(self.latitude, self.longitude), - func.ll_to_earth(*other_location), - ) - - sensor_type = db.relationship( - "WeatherSensorType", backref=db.backref("sensors", lazy=True) - ) - - def __repr__(self): - return "" % ( - self.id, - self.name, - self.weather_sensor_type_name, - self.event_resolution, - ) - - def to_dict(self) -> Dict[str, str]: - return dict(name=self.name, sensor_type=self.weather_sensor_type_name) From 63d4cb7fad2ad27291a28342c003fc16b889b1e7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 13 Jul 2023 14:37:22 +0200 Subject: [PATCH 8/8] delete: remove obsolete WeatherSensorType class and weather module Signed-off-by: F.N. Claessen --- flexmeasures/data/config.py | 1 - flexmeasures/data/models/weather.py | 30 ----------------------------- 2 files changed, 31 deletions(-) diff --git a/flexmeasures/data/config.py b/flexmeasures/data/config.py index 4e2449687..e62240476 100644 --- a/flexmeasures/data/config.py +++ b/flexmeasures/data/config.py @@ -44,7 +44,6 @@ def configure_db_for(app: Flask): from flexmeasures.data.models import ( # noqa: F401 time_series, assets, - weather, data_sources, user, task_runs, diff --git a/flexmeasures/data/models/weather.py b/flexmeasures/data/models/weather.py index 0de4ddfe0..e69de29bb 100644 --- a/flexmeasures/data/models/weather.py +++ b/flexmeasures/data/models/weather.py @@ -1,30 +0,0 @@ -from flexmeasures.data import db -from flexmeasures.data.models.generic_assets import ( - GenericAssetType, -) -from flexmeasures.utils.flexmeasures_inflection import humanize - - -class WeatherSensorType(db.Model): - """ - This model is now considered legacy. See GenericAssetType. - """ - - name = db.Column(db.String(80), primary_key=True) - display_name = db.Column(db.String(80), default="", unique=True) - - daily_seasonality = True - weekly_seasonality = False - yearly_seasonality = True - - def __init__(self, **kwargs): - generic_asset_type = GenericAssetType( - name=kwargs["name"], description=kwargs.get("hover_label", None) - ) - db.session.add(generic_asset_type) - super(WeatherSensorType, self).__init__(**kwargs) - if "display_name" not in kwargs: - self.display_name = humanize(self.name) - - def __repr__(self): - return "" % self.name