Asset audit log

documentation/changelog.rst

@@ -8,6 +8,8 @@ v0.22.0 | June XX, 2024
 
 New features
 -------------
+* Add `asset/<id>/auditlog` to view asset related actions [see `PR #1067 <https://github.com/FlexMeasures/flexmeasures/pull/1067>`_]
+* On the asset page, facilitate comparison by showing the two default sensors together if they record the same unit [see `PR #1066 <https://github.com/FlexMeasures/flexmeasures/pull/1066>`_]
 * Flex-context (price sensors and inflexible device sensors) can now be set on the asset page (and are part of GenericAsset model) [see `PR #1059 <https://github.com/FlexMeasures/flexmeasures/pull/1059/>`_]
 * On the asset page's default view, facilitate comparison by showing the two default sensors together if they record the same unit [see `PR #1066 <https://github.com/FlexMeasures/flexmeasures/pull/1066>`_]
 

flexmeasures/__init__.py

@@ -1,6 +1,7 @@
 from importlib_metadata import version, PackageNotFoundError
 
 from flexmeasures.data.models.annotations import Annotation
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.models.user import (
     Account,
     AccountRole,

flexmeasures/api/v3_0/assets.py

@@ -11,6 +11,7 @@
 from flexmeasures.auth.decorators import permission_required_for_context
 from flexmeasures.data import db
 from flexmeasures.data.models.user import Account
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.schemas import AwareDateTimeField
 from flexmeasures.data.schemas.generic_assets import GenericAssetSchema as AssetSchema
@@ -148,6 +149,9 @@ def post(self, asset_data: dict):
 
         asset.set_inflexible_sensors(inflexible_sensor_ids)
         db.session.commit()
+
+        AssetAuditLog.add_record(asset, f"Created asset '{asset.name}': {asset.id}")
+
         return asset_schema.dump(asset), 201
 
     @route("/<id>", methods=["GET"])
@@ -239,6 +243,24 @@ def patch(self, asset_data: dict, id: int, db_asset: GenericAsset):
         inflexible_sensor_ids = asset_data.pop("inflexible_device_sensor_ids", [])
         db_asset.set_inflexible_sensors(inflexible_sensor_ids)
 
+        audit_log_data = list()
+        for k, v in asset_data.items():
+            if getattr(db_asset, k) == v:
+                continue
+            if k == "attributes":
+                current_attributes = getattr(db_asset, k)
+                for attr_key, attr_value in v.items():
+                    if current_attributes.get(attr_key) != attr_value:
+                        audit_log_data.append(
+                            f"Attribute name: {attr_key}, Old value: {current_attributes.get(attr_key)}, New value: {attr_value}"
+                        )
+                continue
+            audit_log_data.append(
+                f"Field name: {k}, Old value: {getattr(db_asset, k)}, New value: {v}"
+            )
+        audit_log_event = f"Updated asset '{db_asset.name}': {db_asset.id} fields: {'; '.join(audit_log_data)}"
+        AssetAuditLog.add_record(db_asset, audit_log_event)
+
         for k, v in asset_data.items():
             setattr(db_asset, k, v)
         db.session.add(db_asset)
@@ -265,7 +287,9 @@ def delete(self, id: int, asset: GenericAsset):
         :status 403: INVALID_SENDER
         :status 422: UNPROCESSABLE_ENTITY
         """
-        asset_name = asset.name
+        asset_name, asset_id = asset.name, asset.id
+        AssetAuditLog.add_record(asset, f"Deleted asset '{asset_name}': {asset_id}")
+
         db.session.execute(delete(GenericAsset).filter_by(id=asset.id))
         db.session.commit()
         current_app.logger.info("Deleted asset '%s'." % asset_name)
@@ -326,3 +350,49 @@ def get_chart_data(self, id: int, asset: GenericAsset, **kwargs):
         """
         sensors = flatten_unique(asset.sensors_to_show)
         return asset.search_beliefs(sensors=sensors, as_json=True, **kwargs)
+
+    @route("/<id>/auditlog")
+    @use_kwargs(
+        {"asset": AssetIdField(data_key="id")},
+        location="path",
+    )
+    @permission_required_for_context("read", ctx_arg_name="asset")
+    @as_json
+    def auditlog(self, id: int, asset: GenericAsset):
+        """API endpoint to get history of asset related actions.
+        **Example response**
+
+        .. sourcecode:: json
+            [
+                {
+                    'event': 'Asset test asset deleted',
+                    'event_datetime': '2021-01-01T00:00:00',
+                    'active_user_name': 'Test user',
+                }
+            ]
+
+        :reqheader Authorization: The authentication token
+        :reqheader Content-Type: application/json
+        :resheader Content-Type: application/json
+        :status 200: PROCESSED
+        :status 400: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS
+        :status 401: UNAUTHORIZED
+        :status 403: INVALID_SENDER
+        :status 422: UNPROCESSABLE_ENTITY
+        """
+        audit_logs = (
+            db.session.query(AssetAuditLog).filter_by(affected_asset_id=asset.id).all()
+        )
+        audit_logs = [
+            {
+                k: getattr(log, k)
+                for k in (
+                    "event",
+                    "event_datetime",
+                    "active_user_name",
+                    "active_user_id",
+                )
+            }
+            for log in audit_logs
+        ]
+        return audit_logs, 200

flexmeasures/api/v3_0/sensors.py

@@ -31,6 +31,7 @@
 from flexmeasures.api.common.utils.api_utils import save_and_enqueue
 from flexmeasures.auth.decorators import permission_required_for_context
 from flexmeasures.data import db
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.models.user import Account
 from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.models.time_series import Sensor, TimedBelief
@@ -615,7 +616,7 @@ def fetch_one(self, id, sensor):
         pass_ctx_to_loader=True,
     )
     def post(self, sensor_data: dict):
-        """Create new asset.
+        """Create new sensor.
 
         .. :quickref: Sensor; Create a new Sensor
 
@@ -660,6 +661,10 @@ def post(self, sensor_data: dict):
         sensor = Sensor(**sensor_data)
         db.session.add(sensor)
         db.session.commit()
+
+        asset = sensor_schema.context["generic_asset"]
+        AssetAuditLog.add_record(asset, f"Created sensor '{sensor.name}': {sensor.id}")
+
         return sensor_schema.dump(sensor), 201
 
     @route("/<id>", methods=["PATCH"])
@@ -715,6 +720,16 @@ def patch(self, sensor_data: dict, id: int, sensor: Sensor):
         :status 403: INVALID_SENDER
         :status 422: UNPROCESSABLE_ENTITY
         """
+        audit_log_data = list()
+        for k, v in sensor_data.items():
+            if getattr(sensor, k) != v:
+                audit_log_data.append(
+                    f"Field name: {k}, Old value: {getattr(sensor, k)}, New value: {v}"
+                )
+        audit_log_event = f"Updated sensor '{sensor.name}': {sensor.id}. Updated fields: {'; '.join(audit_log_data)}"
+
+        AssetAuditLog.add_record(sensor.generic_asset, audit_log_event)
+
         for k, v in sensor_data.items():
             setattr(sensor, k, v)
         db.session.add(sensor)
@@ -745,6 +760,10 @@ def delete(self, id: int, sensor: Sensor):
         """Delete time series data."""
         db.session.execute(delete(TimedBelief).filter_by(sensor_id=sensor.id))
 
+        AssetAuditLog.add_record(
+            sensor.generic_asset, f"Deleted sensor '{sensor.name}': {sensor.id}"
+        )
+
         sensor_name = sensor.name
         db.session.execute(delete(Sensor).filter_by(id=sensor.id))
         db.session.commit()

flexmeasures/api/v3_0/tests/test_assets_api.py

@@ -4,6 +4,7 @@
 import pytest
 from sqlalchemy import select, func
 
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.models.generic_assets import GenericAsset
 from flexmeasures.data.services.users import find_user_by_email
 from flexmeasures.api.tests.utils import get_auth_token, UserContext, AccountContext
@@ -147,7 +148,9 @@ def test_get_public_assets(
 @pytest.mark.parametrize(
     "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True
 )
-def test_alter_an_asset(client, setup_api_test_data, setup_accounts, requesting_user):
+def test_alter_an_asset(
+    client, setup_api_test_data, setup_accounts, requesting_user, db
+):
     # without being an account-admin, no asset can be created ...
     with AccountContext("Test Prosumer Account") as prosumer:
         prosumer_asset = prosumer.generic_assets[0]
@@ -165,15 +168,38 @@ def test_alter_an_asset(client, setup_api_test_data, setup_accounts, requesting_
     print(f"Deletion Response: {asset_delete_response.json}")
     assert asset_delete_response.status_code == 403
     # ... but editing is allowed.
+    latitude, name = prosumer_asset.latitude, prosumer_asset.name
     asset_edit_response = client.patch(
         url_for("AssetAPI:patch", id=prosumer_asset.id),
         json={
-            "latitude": prosumer_asset.latitude,
-        },  # we're not changing values to keep other tests clean here
+            "latitude": 11.1,
+            "name": "other",
+        },
     )
     print(f"Editing Response: {asset_edit_response.json}")
     assert asset_edit_response.status_code == 200
 
+    # Resetting changes to keep other tests clean here
+    asset_edit_response = client.patch(
+        url_for("AssetAPI:patch", id=prosumer_asset.id),
+        json={
+            "latitude": latitude,
+            "name": name,
+        },
+    )
+    print(f"Editing Response: {asset_edit_response.json}")
+    assert asset_edit_response.status_code == 200
+
+    audit_log_event = f"Updated asset '{prosumer_asset.name}': {prosumer_asset.id} fields: Field name: name, Old value: {name}, New value: other; Field name: latitude, Old value: {latitude}, New value: 11.1"
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            event=audit_log_event,
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+            affected_asset_id=prosumer_asset.id,
+        )
+    ).scalar_one_or_none()
+
 
 @pytest.mark.parametrize(
     "bad_json_str, error_msg",
@@ -353,10 +379,20 @@ def test_post_an_asset(client, setup_api_test_data, requesting_user, db):
     assert asset is not None
     assert asset.latitude == 30.1
 
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            affected_asset_id=asset.id,
+            event=f"Created asset '{asset.name}': {asset.id}",
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+        )
+    ).scalar_one_or_none()
+
 
 @pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True)
 def test_delete_an_asset(client, setup_api_test_data, requesting_user, db):
-    existing_asset_id = setup_api_test_data["some gas sensor"].generic_asset.id
+    existing_asset = setup_api_test_data["some gas sensor"].generic_asset
+    existing_asset_id, existing_asset_name = existing_asset.id, existing_asset.name
 
     delete_asset_response = client.delete(
         url_for("AssetAPI:delete", id=existing_asset_id),
@@ -367,6 +403,15 @@ def test_delete_an_asset(client, setup_api_test_data, requesting_user, db):
     ).scalar_one_or_none()
     assert deleted_asset is None
 
+    audit_log = db.session.execute(
+        select(AssetAuditLog).filter_by(
+            event=f"Deleted asset '{existing_asset_name}': {existing_asset_id}",
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+        )
+    ).scalar_one_or_none()
+    assert audit_log.affected_asset_id is None
+
 
 @pytest.mark.parametrize(
     "requesting_user",

flexmeasures/api/v3_0/tests/test_sensors_api.py

@@ -8,6 +8,7 @@
 from flexmeasures.data.models.time_series import TimedBelief
 from flexmeasures import Sensor
 from flexmeasures.api.v3_0.tests.utils import get_sensor_post_data
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.schemas.sensors import SensorSchema
 from flexmeasures.data.models.generic_assets import GenericAsset
 
@@ -84,6 +85,15 @@ def test_post_a_sensor(client, setup_api_test_data, requesting_user, db):
     assert sensor.unit == "kWh"
     assert sensor.attributes["capacity_in_mw"] == 0.0074
 
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            affected_asset_id=post_data["generic_asset_id"],
+            event=f"Created sensor '{sensor.name}': {sensor.id}",
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+        )
+    ).scalar_one_or_none()
+
 
 @pytest.mark.parametrize(
     "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True
@@ -131,6 +141,19 @@ def test_patch_sensor(client, setup_api_test_data, requesting_user, db):
     )
     assert new_sensor.attributes["test_attribute"] == "test_attribute_value"
 
+    audit_log_event = (
+        f"Updated sensor 'some gas sensor': {sensor.id}. Updated fields: Field name: name, Old value: some gas sensor, New value: Changed name; Field name: attributes, "
+        + "Old value: {}, New value: {'test_attribute': 'test_attribute_value'}"
+    )
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            event=audit_log_event,
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+            affected_asset_id=sensor.generic_asset_id,
+        )
+    ).scalar_one_or_none()
+
 
 @pytest.mark.parametrize(
     "attribute, value",
@@ -184,7 +207,8 @@ def test_patch_sensor_non_admin(client, setup_api_test_data, requesting_user, db
 
 @pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True)
 def test_delete_a_sensor(client, setup_api_test_data, requesting_user, db):
-    existing_sensor_id = setup_api_test_data["some temperature sensor"].id
+    existing_sensor = setup_api_test_data["some temperature sensor"]
+    existing_sensor_id = existing_sensor.id
     sensor_data = db.session.scalars(
         select(TimedBelief).filter(TimedBelief.sensor_id == existing_sensor_id)
     ).all()
@@ -207,3 +231,12 @@ def test_delete_a_sensor(client, setup_api_test_data, requesting_user, db):
     assert (
         db.session.scalar(select(func.count()).select_from(Sensor)) == sensor_count - 1
     )
+
+    assert db.session.execute(
+        select(AssetAuditLog).filter_by(
+            affected_asset_id=existing_sensor.generic_asset_id,
+            event=f"Deleted sensor '{existing_sensor.name}': {existing_sensor.id}",
+            active_user_id=requesting_user.id,
+            active_user_name=requesting_user.username,
+        )
+    ).scalar_one_or_none()

flexmeasures/cli/data_edit.py

@@ -21,6 +21,7 @@
 from flexmeasures.data.schemas.generic_assets import GenericAssetIdField
 from flexmeasures.data.schemas.sensors import SensorIdField
 from flexmeasures.data.models.generic_assets import GenericAsset
+from flexmeasures.data.models.audit_log import AssetAuditLog
 from flexmeasures.data.models.time_series import TimedBelief
 from flexmeasures.data.utils import save_to_db
 from flexmeasures.cli.utils import MsgStyle, DeprecatedOption, DeprecatedOptionsCommand
@@ -146,9 +147,15 @@ def edit_attribute(
 
     # Set attribute
     for asset in assets:
+        AssetAuditLog.add_record_for_attribute_update(
+            attribute_key, attribute_value, "asset", asset
+        )
         asset.attributes[attribute_key] = attribute_value
         db.session.add(asset)
     for sensor in sensors:
+        AssetAuditLog.add_record_for_attribute_update(
+            attribute_key, attribute_value, "sensor", sensor
+        )
         sensor.attributes[attribute_key] = attribute_value
         db.session.add(sensor)
     db.session.commit()
@@ -228,6 +235,11 @@ def resample_sensor_data(
                 abort=True,
             )
 
+        AssetAuditLog.add_record(
+            sensor.generic_asset,
+            f"Resampled sensor data for sensor '{sensor.name}': {sensor.id} to {event_resolution} from {sensor.event_resolution}",
+        )
+
         # Update sensor
         sensor.event_resolution = event_resolution
         db.session.add(sensor)
@@ -268,6 +280,11 @@ def transfer_ownership(asset: Asset, new_owner: Account):
     """
 
     def transfer_ownership_recursive(asset: Asset, account: Account):
+        AssetAuditLog.add_record(
+            asset,
+            f"Transfered ownership for asset '{asset.name}': {asset.id} from '{asset.owner.name}': {asset.owner.id} to '{account.name}': {account.id}",
+        )
+
         asset.owner = account
         for child in asset.child_assets:
             transfer_ownership_recursive(child, account)