Merge branch 'main' into 763-requirements

.vscode/spellright.dict

@@ -257,3 +257,8 @@ dataframe
 dataframes
 args
 docstrings
+Auth
+ctx_loader
+ctx_arg_name
+ctx_arg_pos
+dataset

documentation/api/change_log.rst

@@ -5,6 +5,12 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace.
 
+
+v3.0-12 | 2023-07-31
+"""""""""""""""""""
+
+- Added REST endpoint for adding a sensor: `/sensors` (POST)
+
 v3.0-11 | 2023-07-20
 """"""""""""""""""""
 

documentation/changelog.rst

@@ -8,14 +8,17 @@ v0.15.0 | July XX, 2023
 
 .. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``).
 
+.. warning:: If your server is running in play mode (``FLEXMEASURES_MODE = "play"``), users will be able to see sensor data from any account [see `PR #740 <https://www.github.com/FlexMeasures/flexmeasures/pull/740>`_].
+
 New features
 -------------
 
 * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times [see `PR #734 <https://www.github.com/FlexMeasures/flexmeasures/pull/734>`_]
 * Make it a lot easier to read off the color legend on the asset page, especially when showing many sensors, as they will now be ordered from top to bottom in the same order as they appear in the chart (as defined in the ``sensors_to_show`` attribute), rather than alphabetically [see `PR #742 <https://www.github.com/FlexMeasures/flexmeasures/pull/742>`_]
+* Users on FlexMeasures servers in play mode (``FLEXMEASURES_MODE = "play"``) can use the ``sensors_to_show`` attribute to show any sensor on their asset pages, rather than only sensors registered to assets in their own account or to public assets [see `PR #740 <https://www.github.com/FlexMeasures/flexmeasures/pull/740>`_]
 * Having percentages within the [0, 100] domain is such a common use case that we now always include it in sensor charts with % units, making it easier to read off individual charts and also to compare across charts [see `PR #739 <https://www.github.com/FlexMeasures/flexmeasures/pull/739>`_]
 * DataSource table now allows storing arbitrary attributes as a JSON (without content validation), similar to the Sensor and GenericAsset tables [see `PR #750 <https://www.github.com/FlexMeasures/flexmeasures/pull/750>`_]
-* Added API endpoint `/sensor/<id>` for fetching a single sensor. [see `PR #759 <https://www.github.com/FlexMeasures/flexmeasures/pull/759>`_]
+* Added API endpoints `/sensors/<id>` for fetching a single sensor and `/sensors` (POST) for adding a sensor. [see `PR #759 <https://www.github.com/FlexMeasures/flexmeasures/pull/759>`_] and [see `PR #767 <https://www.github.com/FlexMeasures/flexmeasures/pull/767>`_]
 * The CLI now allows to set lists and dicts as asset & sensor attributes (formerly only single values) [see `PR #762 <https://www.github.com/FlexMeasures/flexmeasures/pull/762>`_]
 * Add `ProcessScheduler` class, which optimizes the starting time of processes using one of the following policies: INFLEXIBLE, SHIFTABLE and BREAKABLE [see `PR #729 <https://www.github.com/FlexMeasures/flexmeasures/pull/729>`_]
 

documentation/dev/auth.rst

@@ -30,7 +30,7 @@ You, as the endpoint author, need to make sure this is checked. Here is an examp
         {"the_resource": ResourceIdField(data_key="resource_id")},
         location="path",
     )
-    @permission_required_for_context("read", arg_name="the_resource")
+    @permission_required_for_context("read", ctx_arg_name="the_resource")
     @as_json
     def view(resource_id: int, resource: Resource):
         return dict(name=resource.name)

documentation/host/modes.rst

@@ -39,3 +39,4 @@ Small features
 - [API] Posted UDI events are not enforced to be consecutive.
 - [API] Names in ``GetConnectionResponse`` are the connections' unique database names rather than their display names (this feature is planned to be deprecated).
 - [UI] The dashboard plot showing the latest power value is not enforced to lie in the past (in case of simulating future values).
+- [UI] On the asset page, the ``sensors_to_show`` attribute can be used to show any sensor from any account, rather than only sensors from assets owned by the user's organization.

flexmeasures/api/dev/sensors.py

@@ -50,7 +50,7 @@ class SensorAPI(FlaskView):
         },
         location="query",
     )
-    @permission_required_for_context("read", arg_name="sensor")
+    @permission_required_for_context("read", ctx_arg_name="sensor")
     def get_chart(self, id: int, sensor: Sensor, **kwargs):
         """GET from /sensor/<id>/chart
 
@@ -85,7 +85,7 @@ def get_chart(self, id: int, sensor: Sensor, **kwargs):
         },
         location="query",
     )
-    @permission_required_for_context("read", arg_name="sensor")
+    @permission_required_for_context("read", ctx_arg_name="sensor")
     def get_chart_data(self, id: int, sensor: Sensor, **kwargs):
         """GET from /sensor/<id>/chart_data
 
@@ -118,7 +118,7 @@ def get_chart_data(self, id: int, sensor: Sensor, **kwargs):
         },
         location="query",
     )
-    @permission_required_for_context("read", arg_name="sensor")
+    @permission_required_for_context("read", ctx_arg_name="sensor")
     def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs):
         """GET from /sensor/<id>/chart_annotations
 
@@ -147,7 +147,7 @@ def get_chart_annotations(self, id: int, sensor: Sensor, **kwargs):
         {"sensor": SensorIdField(data_key="id")},
         location="path",
     )
-    @permission_required_for_context("read", arg_name="sensor")
+    @permission_required_for_context("read", ctx_arg_name="sensor")
     def get(self, id: int, sensor: Sensor):
         """GET from /sensor/<id>
 
@@ -170,7 +170,7 @@ class AssetAPI(FlaskView):
         {"asset": AssetIdField(data_key="id")},
         location="path",
     )
-    @permission_required_for_context("read", arg_name="asset")
+    @permission_required_for_context("read", ctx_arg_name="asset")
     def get(self, id: int, asset: GenericAsset):
         """GET from /asset/<id>
 

flexmeasures/api/v3_0/accounts.py

@@ -70,7 +70,7 @@ def index(self):
 
     @route("/<id>", methods=["GET"])
     @use_kwargs({"account": AccountIdField(data_key="id")}, location="path")
-    @permission_required_for_context("read", arg_name="account")
+    @permission_required_for_context("read", ctx_arg_name="account")
     @as_json
     def get(self, id: int, account: Account):
         """API endpoint to get an account.

flexmeasures/api/v3_0/assets.py

@@ -42,7 +42,7 @@ class AssetAPI(FlaskView):
         },
         location="query",
     )
-    @permission_required_for_context("read", arg_name="account")
+    @permission_required_for_context("read", ctx_arg_name="account")
     @as_json
     def index(self, account: Account):
         """List all assets owned by a certain account.
@@ -103,7 +103,7 @@ def public(self):
 
     @route("", methods=["POST"])
     @permission_required_for_context(
-        "create-children", arg_loader=AccountIdField.load_current
+        "create-children", ctx_loader=AccountIdField.load_current
     )
     @use_args(asset_schema)
     def post(self, asset_data: dict):
@@ -144,7 +144,7 @@ def post(self, asset_data: dict):
 
     @route("/<id>", methods=["GET"])
     @use_kwargs({"asset": AssetIdField(data_key="id")}, location="path")
-    @permission_required_for_context("read", arg_name="asset")
+    @permission_required_for_context("read", ctx_arg_name="asset")
     @as_json
     def fetch_one(self, id, asset):
         """Fetch a given asset.
@@ -180,7 +180,7 @@ def fetch_one(self, id, asset):
     @route("/<id>", methods=["PATCH"])
     @use_args(partial_asset_schema)
     @use_kwargs({"db_asset": AssetIdField(data_key="id")}, location="path")
-    @permission_required_for_context("update", arg_name="db_asset")
+    @permission_required_for_context("update", ctx_arg_name="db_asset")
     @as_json
     def patch(self, asset_data: dict, id: int, db_asset: GenericAsset):
         """Update an asset given its identifier.
@@ -236,7 +236,7 @@ def patch(self, asset_data: dict, id: int, db_asset: GenericAsset):
 
     @route("/<id>", methods=["DELETE"])
     @use_kwargs({"asset": AssetIdField(data_key="id")}, location="path")
-    @permission_required_for_context("delete", arg_name="asset")
+    @permission_required_for_context("delete", ctx_arg_name="asset")
     @as_json
     def delete(self, id: int, asset: GenericAsset):
         """Delete an asset given its identifier.
@@ -278,7 +278,7 @@ def delete(self, id: int, asset: GenericAsset):
         },
         location="query",
     )
-    @permission_required_for_context("read", arg_name="asset")
+    @permission_required_for_context("read", ctx_arg_name="asset")
     def get_chart(self, id: int, asset: GenericAsset, **kwargs):
         """GET from /assets/<id>/chart
 
@@ -302,7 +302,7 @@ def get_chart(self, id: int, asset: GenericAsset, **kwargs):
         },
         location="query",
     )
-    @permission_required_for_context("read", arg_name="asset")
+    @permission_required_for_context("read", ctx_arg_name="asset")
     def get_chart_data(self, id: int, asset: GenericAsset, **kwargs):
         """GET from /assets/<id>/chart_data
 

flexmeasures/api/v3_0/sensors.py

@@ -30,6 +30,7 @@
 from flexmeasures.auth.decorators import permission_required_for_context
 from flexmeasures.data import db
 from flexmeasures.data.models.user import Account
+from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.models.time_series import Sensor
 from flexmeasures.data.queries.utils import simplify_index
 from flexmeasures.data.schemas.sensors import SensorSchema, SensorIdField
@@ -64,7 +65,7 @@ class SensorAPI(FlaskView):
         },
         location="query",
     )
-    @permission_required_for_context("read", arg_name="account")
+    @permission_required_for_context("read", ctx_arg_name="account")
     @as_json
     def index(self, account: Account):
         """API endpoint to list all sensors of an account.
@@ -498,7 +499,7 @@ def get_schedule(self, sensor: Sensor, job_id: str, duration: timedelta, **kwarg
 
     @route("/<id>", methods=["GET"])
     @use_kwargs({"sensor": SensorIdField(data_key="id")}, location="path")
-    @permission_required_for_context("read", arg_name="sensor")
+    @permission_required_for_context("read", ctx_arg_name="sensor")
     @as_json
     def fetch_one(self, id, sensor):
         """Fetch a given sensor.
@@ -515,7 +516,7 @@ def fetch_one(self, id, sensor):
                 "name": "some gas sensor",
                 "unit": "m³/h",
                 "entity_address": "ea1.2023-08.localhost:fm1.1",
-                "event_resolution": 10,
+                "event_resolution": "PT10M",
                 "generic_asset_id": 4,
                 "timezone": "UTC",
             }
@@ -529,4 +530,49 @@ def fetch_one(self, id, sensor):
         :status 403: INVALID_SENDER
         :status 422: UNPROCESSABLE_ENTITY
         """
+
         return sensor_schema.dump(sensor), 200
+
+    @route("", methods=["POST"])
+    @use_args(sensor_schema)
+    @permission_required_for_context(
+        "create-children",
+        ctx_arg_pos=1,
+        ctx_arg_name="generic_asset_id",
+        ctx_loader=GenericAsset,
+        pass_ctx_to_loader=True,
+    )
+    def post(self, sensor_data: dict):
+        """Create new asset.
+
+        .. :quickref: Sensor; Create a new Sensor
+
+        This endpoint creates a new Sensor.
+
+        **Example request**
+
+        .. sourcecode:: json
+
+            {
+                "name": "power",
+                "event_resolution": "PT1H",
+                "unit": "kWh",
+                "generic_asset_id": 1,
+            }
+
+
+        The newly posted sensor is returned in the response.
+
+        :reqheader Authorization: The authentication token
+        :reqheader Content-Type: application/json
+        :resheader Content-Type: application/json
+        :status 201: CREATED
+        :status 400: INVALID_REQUEST
+        :status 401: UNAUTHORIZED
+        :status 403: INVALID_SENDER
+        :status 422: UNPROCESSABLE_ENTITY
+        """
+        sensor = Sensor(**sensor_data)
+        db.session.add(sensor)
+        db.session.commit()
+        return sensor_schema.dump(sensor), 201

flexmeasures/api/v3_0/tests/test_sensors_api.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
-
+import pytest
 from flask import url_for
 
 
 from flexmeasures import Sensor
 from flexmeasures.api.tests.utils import get_auth_token
+from flexmeasures.api.v3_0.tests.utils import get_sensor_post_data
+from flexmeasures.data.schemas.sensors import SensorSchema
+
+sensor_schema = SensorSchema()
 
 
 def test_fetch_one_sensor(
@@ -24,10 +28,81 @@ def test_fetch_one_sensor(
     assert response.json["unit"] == "m³/h"
     assert response.json["generic_asset_id"] == 4
     assert response.json["timezone"] == "UTC"
+    assert response.json["event_resolution"] == "PT10M"
+
+
+@pytest.mark.parametrize("use_auth", [False, True])
+def test_fetch_one_sensor_no_auth(
+    client, setup_api_test_data: dict[str, Sensor], use_auth
+):
+    """Test 1: Sensor with id 1 is not in the test_prosumer_user_2@seita.nl's account.
+    The Supplier Account as can be seen in flexmeasures/api/v3_0/tests/conftest.py
+    Test 2: There is no authentication int the headers"""
+    sensor_id = 1
+    if use_auth:
+        headers = make_headers_for("test_prosumer_user_2@seita.nl", client)
+        response = client.get(
+            url_for("SensorAPI:fetch_one", id=sensor_id),
+            headers=headers,
+        )
+        assert response.status_code == 403
+        assert (
+            response.json["message"]
+            == "You cannot be authorized for this content or functionality."
+        )
+        assert response.json["status"] == "INVALID_SENDER"
+    else:
+        headers = make_headers_for(None, client)
+        response = client.get(
+            url_for("SensorAPI:fetch_one", id=sensor_id),
+            headers=headers,
+        )
+        assert response.status_code == 401
+        assert (
+            response.json["message"]
+            == "You could not be properly authenticated for this content or functionality."
+        )
+        assert response.json["status"] == "UNAUTHORIZED"
 
 
 def make_headers_for(user_email: str | None, client) -> dict:
     headers = {"content-type": "application/json"}
     if user_email:
         headers["Authorization"] = get_auth_token(client, user_email, "testtest")
     return headers
+
+
+def test_post_a_sensor(client, setup_api_test_data):
+    auth_token = get_auth_token(client, "test_admin_user@seita.nl", "testtest")
+    post_data = get_sensor_post_data()
+    response = client.post(
+        url_for("SensorAPI:post"),
+        json=post_data,
+        headers={"content-type": "application/json", "Authorization": auth_token},
+    )
+    print("Server responded with:\n%s" % response.json)
+    assert response.status_code == 201
+    assert response.json["name"] == "power"
+    assert response.json["event_resolution"] == "PT1H"
+
+    sensor: Sensor = Sensor.query.filter_by(name="power").one_or_none()
+    assert sensor is not None
+    assert sensor.unit == "kWh"
+
+
+def test_post_sensor_to_asset_from_unrelated_account(client, setup_api_test_data):
+    """Tries to add sensor to account the user doesn't have access to"""
+    auth_token = get_auth_token(client, "test_supplier_user_4@seita.nl", "testtest")
+    post_data = get_sensor_post_data()
+    response = client.post(
+        url_for("SensorAPI:post"),
+        json=post_data,
+        headers={"content-type": "application/json", "Authorization": auth_token},
+    )
+    print("Server responded with:\n%s" % response.json)
+    assert response.status_code == 403
+    assert (
+        response.json["message"]
+        == "You cannot be authorized for this content or functionality."
+    )
+    assert response.json["status"] == "INVALID_SENDER"

flexmeasures/api/v3_0/tests/utils.py

@@ -40,6 +40,16 @@ def get_asset_post_data(account_id: int = 1, asset_type_id: int = 1) -> dict:
     return post_data
 
 
+def get_sensor_post_data(generic_asset_id: int = 2) -> dict:
+    post_data = {
+        "name": "power",
+        "event_resolution": "PT1H",
+        "unit": "kWh",
+        "generic_asset_id": generic_asset_id,
+    }
+    return post_data
+
+
 def message_for_trigger_schedule(
     unknown_prices: bool = False,
     with_targets: bool = False,