Asset audit log

Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com>

documentation/changelog.rst

@@ -11,6 +11,7 @@ New features
 
 * Add option `droplevels` to the `PandasReporter` to drop all the levels except the `event_start` and `event_value` [see `PR #1043 <https://github.com/FlexMeasures/flexmeasures/pull/1043/>`_]
 * `PandasReporter` accepts the parameter `use_latest_version_only` to filter input data [see `PR #1045 <https://github.com/FlexMeasures/flexmeasures/pull/1045/>`_]
+* Add `asset/<id>/auditlog` to view asset related actions [see `PR #1067 <https://github.com/FlexMeasures/flexmeasures/pull/1067>`_]
 
 
 v0.21.0 | April 16, 2024

flexmeasures/api/v3_0/assets.py

@@ -2,7 +2,7 @@
 
 from flask import current_app
 from flask_classful import FlaskView, route
-from flask_security import auth_required
+from flask_security import auth_required, current_user
 from flask_json import as_json
 from marshmallow import fields
 from webargs.flaskparser import use_kwargs, use_args
@@ -11,12 +11,14 @@
 from flexmeasures.auth.decorators import permission_required_for_context
 from flexmeasures.data import db
 from flexmeasures.data.models.user import Account
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.schemas import AwareDateTimeField
 from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema
 from flexmeasures.api.common.schemas.generic_assets import AssetIdField
 from flexmeasures.api.common.schemas.users import AccountIdField
 from flexmeasures.utils.coding_utils import flatten_unique
+from flexmeasures.utils.time_utils import server_now
 from flexmeasures.ui.utils.view_utils import set_session_variables
 
 
@@ -143,6 +145,16 @@ def post(self, asset_data: dict):
         asset = GenericAsset(**asset_data)
         db.session.add(asset)
         db.session.commit()
+
+        audit_log = AssetAuditLog(
+            event_datetime=server_now(),
+            event=f"Created asset '{asset.name}': {asset.id}",
+            active_user_id=current_user.id,
+            active_user_name=current_user.username,
+            affected_asset_id=asset.id,
+        )
+        db.session.add(audit_log)
+
         return asset_schema.dump(asset), 201
 
     @route("/<id>", methods=["GET"])
@@ -231,6 +243,23 @@ def patch(self, asset_data: dict, id: int, db_asset: GenericAsset):
         :status 403: INVALID_SENDER
         :status 422: UNPROCESSABLE_ENTITY
         """
+
+        audit_log_data = list()
+        for k, v in asset_data.items():
+            audit_log_data.append(
+                f"Field name: {k}, Old value: {getattr(db_asset, k)}, New value: {v}"
+            )
+        audit_log_event = f"Updated asset '{db_asset.name}': {db_asset.id}. Updated fields: {'; '.join(audit_log_data)}"
+
+        audit_log = AssetAuditLog(
+            event_datetime=server_now(),
+            event=audit_log_event,
+            active_user_id=current_user.id,
+            active_user_name=current_user.username,
+            affected_asset_id=db_asset.id,
+        )
+        db.session.add(audit_log)
+
         for k, v in asset_data.items():
             setattr(db_asset, k, v)
         db.session.add(db_asset)
@@ -257,7 +286,16 @@ def delete(self, id: int, asset: GenericAsset):
         :status 403: INVALID_SENDER
         :status 422: UNPROCESSABLE_ENTITY
         """
-        asset_name = asset.name
+        asset_name, asset_id = asset.name, asset.id
+        audit_log = AssetAuditLog(
+            event_datetime=server_now(),
+            event=f"Deleted asset '{asset_name}': {asset_id}",
+            active_user_id=current_user.id,
+            active_user_name=current_user.username,
+            affected_asset_id=asset_id,
+        )
+        db.session.add(audit_log)
+
         db.session.execute(delete(GenericAsset).filter_by(id=asset.id))
         db.session.commit()
         current_app.logger.info("Deleted asset '%s'." % asset_name)
@@ -318,3 +356,44 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs):
         """
         sensors = flatten_unique(asset.sensors_to_show)
         return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs)
+
+    @route("/<id>/auditlog")
+    @use_kwargs(
+        {"asset": AssetIdField(data_key="id")},
+        location="path",
+    )
+    @permission_required_for_context("read", ctx_arg_name="asset")
+    @as_json
+    def auditlog(self, id: int, asset: GenericAsset):
+        """API endpoint to get history of asset related actions.
+        **Example response**
+
+        .. sourcecode:: json
+            [
+                {
+                    'event': 'Asset test asset deleted',
+                    'event_datetime': '2021-01-01T00:00:00',
+                    'active_user_name': 'Test user',
+                }
+            ]
+
+        :reqheader Authorization: The authentication token
+        :reqheader Content-Type: application/json
+        :resheader Content-Type: application/json
+        :status 200: PROCESSED
+        :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS
+        :status 401: UNAUTHORIZED
+        :status 403: INVALID_SENDER
+        :status 422: UNPROCESSABLE_ENTITY
+        """
+        audit_logs = (
+            db.session.query(AssetAuditLog).filter_by(affected_asset_id=asset.id).all()
+        )
+        audit_logs = [
+            {
+                k: getattr(log, k)
+                for k in ("event", "event_datetime", "active_user_name")
+            }
+            for log in audit_logs
+        ]
+        return audit_logs, 200

flexmeasures/api/v3_0/sensors.py

@@ -5,7 +5,7 @@
 from flask import current_app, url_for
 from flask_classful import FlaskView, route
 from flask_json import as_json
-from flask_security import auth_required
+from flask_security import auth_required, current_user
 import isodate
 from marshmallow import fields, ValidationError
 from rq.job import Job, NoSuchJobError
@@ -31,6 +31,7 @@
 from flexmeasures.api.common.utils.api_utils import save_and_enqueue
 from flexmeasures.auth.decorators import permission_required_for_context
 from flexmeasures.data import db
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.models.user import Account
 from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
@@ -42,7 +43,7 @@
     create_scheduling_job,
     get_data_source_for_job,
 )
-from flexmeasures.utils.time_utils import duration_isoformat
+from flexmeasures.utils.time_utils import duration_isoformat, server_now
 
 
 # Instantiate schemas outside of endpoint logic to minimize response time
@@ -612,7 +613,7 @@ def fetch_one(self, id, sensor):
         pass_ctx_to_loader=True,
     )
     def post(self, sensor_data: dict):
-        """Create new asset.
+        """Create new sensor.
 
         .. :quickref: Sensor; Create a new Sensor
 
@@ -657,6 +658,17 @@ def post(self, sensor_data: dict):
         sensor = Sensor(**sensor_data)
         db.session.add(sensor)
         db.session.commit()
+
+        asset = sensor_schema.context["generic_asset"]
+        audit_log = AssetAuditLog(
+            event_datetime=server_now(),
+            event=f"Created sensor '{sensor.name}': {sensor.id}",
+            active_user_id=current_user.id,
+            active_user_name=current_user.username,
+            affected_asset_id=asset.id,
+        )
+        db.session.add(audit_log)
+
         return sensor_schema.dump(sensor), 201
 
     @route("/<id>", methods=["PATCH"])
@@ -712,6 +724,22 @@ def patch(self, sensor_data: dict, id: int, sensor: Sensor):
         :status 403: INVALID_SENDER
         :status 422: UNPROCESSABLE_ENTITY
         """
+        audit_log_data = list()
+        for k, v in sensor_data.items():
+            audit_log_data.append(
+                f"Field name: {k}, Old value: {getattr(sensor, k)}, New value: {v}"
+            )
+        audit_log_event = f"Updated sensor '{sensor.name}': {sensor.id}. Updated fields: {'; '.join(audit_log_data)}"
+
+        audit_log = AssetAuditLog(
+            event_datetime=server_now(),
+            event=audit_log_event,
+            active_user_id=current_user.id,
+            active_user_name=current_user.username,
+            affected_asset_id=sensor.generic_asset_id,
+        )
+        db.session.add(audit_log)
+
         for k, v in sensor_data.items():
             setattr(sensor, k, v)
         db.session.add(sensor)
@@ -742,6 +770,16 @@ def delete(self, id: int, sensor: Sensor):
         """Delete time series data."""
         db.session.execute(delete(TimedBelief).filter_by(sensor_id=sensor.id))
 
+        asset = sensor.generic_asset
+        audit_log = AssetAuditLog(
+            event_datetime=server_now(),
+            event=f"Deleted sensor '{sensor.name}': {sensor.id}",
+            active_user_id=current_user.id,
+            active_user_name=current_user.username,
+            affected_asset_id=asset.id,
+        )
+        db.session.add(audit_log)
+
         sensor_name = sensor.name
         db.session.execute(delete(Sensor).filter_by(id=sensor.id))
         db.session.commit()

flexmeasures/api/v3_0/tests/test_assets_api.py

@@ -4,6 +4,7 @@
 import pytest
 from sqlalchemy import select, func
 
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.services.users import find_user_by_email
 from flexmeasures.api.tests.utils import get_auth_token, UserContext, AccountContext
@@ -147,7 +148,9 @@ def test_get_public_assets(
 @pytest.mark.parametrize(
     "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True
 )
-def test_alter_an_asset(client, setup_api_test_data, setup_accounts, requesting_user):
+def test_alter_an_asset(
+    client, setup_api_test_data, setup_accounts, requesting_user, db
+):
     # without being an account-admin, no asset can be created ...
     with AccountContext("Test Prosumer Account") as prosumer:
         prosumer_asset = prosumer.generic_assets[0]
@@ -165,11 +168,34 @@ def test_alter_an_asset(client, setup_api_test_data, setup_accounts, requesting_
     print(f"Deletion Response: {asset_delete_response.json}")
     assert asset_delete_response.status_code == 403
     # ... but editing is allowed.
+    latitude, name = prosumer_asset.latitude, prosumer_asset.name
     asset_edit_response = client.patch(
         url_for("AssetAPI:patch", id=prosumer_asset.id),
         json={
-            "latitude": prosumer_asset.latitude,
-        },  # we're not changing values to keep other tests clean here
+            "latitude": 11.1,
+            "name": "other",
+        },
+    )
+    print(f"Editing Response: {asset_edit_response.json}")
+    assert asset_edit_response.status_code == 200
+
+    audit_log_event = f"Updated asset '{prosumer_asset.name}': {prosumer_asset.id}. Updated fields: Field name: name, Old value: {name}, New value: other; Field name: latitude, Old value: {latitude}, New value: 11.1"
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            event=audit_log_event,
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+            affected_asset_id=prosumer_asset.id,
+        )
+    ).scalar_one_or_none()
+
+    # Resetting changes to keep other tests clean here
+    asset_edit_response = client.patch(
+        url_for("AssetAPI:patch", id=prosumer_asset.id),
+        json={
+            "latitude": latitude,
+            "name": name,
+        },
     )
     print(f"Editing Response: {asset_edit_response.json}")
     assert asset_edit_response.status_code == 200
@@ -353,10 +379,20 @@ def test_post_an_asset(client, setup_api_test_data, requesting_user, db):
     assert asset is not None
     assert asset.latitude == 30.1
 
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            affected_asset_id=asset.id,
+            event=f"Created asset '{asset.name}': {asset.id}",
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+        )
+    ).scalar_one_or_none()
+
 
 @pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True)
 def test_delete_an_asset(client, setup_api_test_data, requesting_user, db):
-    existing_asset_id = setup_api_test_data["some gas sensor"].generic_asset.id
+    existing_asset = setup_api_test_data["some gas sensor"].generic_asset
+    existing_asset_id, existing_asset_name = existing_asset.id, existing_asset.name
 
     delete_asset_response = client.delete(
         url_for("AssetAPI:delete", id=existing_asset_id),
@@ -367,6 +403,15 @@ def test_delete_an_asset(client, setup_api_test_data, requesting_user, db):
     ).scalar_one_or_none()
     assert deleted_asset is None
 
+    audit_log = db.session.execute(
+        select(AssetAuditLog).filter_by(
+            event=f"Deleted asset '{existing_asset_name}': {existing_asset_id}",
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+        )
+    ).scalar_one_or_none()
+    assert audit_log.affected_asset_id is None
+
 
 @pytest.mark.parametrize(
     "requesting_user",

flexmeasures/api/v3_0/tests/test_sensors_api.py

@@ -8,6 +8,7 @@
 from flexmeasures.data.models.time_series import TimedBelief
 from flexmeasures import Sensor
 from flexmeasures.api.v3_0.tests.utils import get_sensor_post_data
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.schemas.sensors import SensorSchema
 from flexmeasures.data.models.generic_assets import GenericAsset
 
@@ -84,6 +85,15 @@ def test_post_a_sensor(client, setup_api_test_data, requesting_user, db):
     assert sensor.unit == "kWh"
     assert sensor.attributes["capacity_in_mw"] == 0.0074
 
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            affected_asset_id=post_data["generic_asset_id"],
+            event=f"Created sensor '{sensor.name}': {sensor.id}",
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+        )
+    ).scalar_one_or_none()
+
 
 @pytest.mark.parametrize(
     "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True
@@ -131,6 +141,19 @@ def test_patch_sensor(client, setup_api_test_data, requesting_user, db):
     )
     assert new_sensor.attributes["test_attribute"] == "test_attribute_value"
 
+    audit_log_event = (
+        f"Updated sensor 'some gas sensor': {sensor.id}. Updated fields: Field name: name, Old value: some gas sensor, New value: Changed name; Field name: attributes, "
+        + "Old value: {}, New value: {'test_attribute': 'test_attribute_value'}"
+    )
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            event=audit_log_event,
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+            affected_asset_id=sensor.generic_asset_id,
+        )
+    ).scalar_one_or_none()
+
 
 @pytest.mark.parametrize(
     "attribute, value",
@@ -184,7 +207,8 @@ def test_patch_sensor_non_admin(client, setup_api_test_data, requesting_user, db
 
 @pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True)
 def test_delete_a_sensor(client, setup_api_test_data, requesting_user, db):
-    existing_sensor_id = setup_api_test_data["some temperature sensor"].id
+    existing_sensor = setup_api_test_data["some temperature sensor"]
+    existing_sensor_id = existing_sensor.id
     sensor_data = db.session.scalars(
         select(TimedBelief).filter(TimedBelief.sensor_id == existing_sensor_id)
     ).all()
@@ -207,3 +231,12 @@ def test_delete_a_sensor(client, setup_api_test_data, requesting_user, db):
     assert (
         db.session.scalar(select(func.count()).select_from(Sensor)) == sensor_count - 1
     )
+
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            affected_asset_id=existing_sensor.generic_asset_id,
+            event=f"Deleted sensor '{existing_sensor.name}': {existing_sensor.id}",
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+        )
+    ).scalar_one_or_none()