Merge branch 'main' into fix/use-https-correctly-when-calling-the-int…

…ernal-API

documentation/changelog.rst

@@ -19,7 +19,8 @@ New features
 Infrastructure / Support
 ----------------------
 
-
+* Improve processing time for deleting beliefs via CLI [see `PR #1005 <https://github.com/FlexMeasures/flexmeasures/pull/1005>`_]
+* Support deleting beliefs via CLI for all offspring assets at once [see `PR #1003 <https://github.com/FlexMeasures/flexmeasures/pull/1003>`_]
 
 
 v0.19.2 | March 1, 2024

flexmeasures/cli/data_delete.py

@@ -256,11 +256,13 @@ def delete_prognoses(
     required=False,
     help="Remove beliefs about events ending at this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.",
 )
+@click.option("--offspring", type=bool, required=False, default=False, is_flag=True)
 def delete_beliefs(  # noqa: C901
     generic_assets: list[GenericAsset],
     sensors: list[Sensor],
     start: datetime | None = None,
     end: datetime | None = None,
+    offspring: bool = False,
 ):
     """Delete all beliefs recorded on a given sensor (or on sensors of a given asset)."""
 
@@ -271,6 +273,8 @@ def delete_beliefs(  # noqa: C901
         abort("Passing both sensors and assets at the same time is not supported.")
     if start is not None and end is not None and start > end:
         abort("Start should not exceed end.")
+    if offspring and len(generic_assets) == 0:
+        abort("Must pass at least one asset when the offspring option is employed.")
 
     # Time window filter
     event_filters = []
@@ -284,6 +288,14 @@ def delete_beliefs(  # noqa: C901
     if sensors:
         entity_filters += [TimedBelief.sensor_id.in_([sensor.id for sensor in sensors])]
     if generic_assets:
+
+        # get the offspring of all generic assets
+        generic_assets_offspring = []
+
+        for asset in generic_assets:
+            generic_assets_offspring.extend(asset.offspring)
+        generic_assets = list(generic_assets) + generic_assets_offspring
+
         entity_filters += [
             TimedBelief.sensor_id == Sensor.id,
             Sensor.generic_asset_id.in_([asset.id for asset in generic_assets]),
@@ -300,26 +312,22 @@ def delete_beliefs(  # noqa: C901
     elif generic_assets:
         prompt = f"Delete all {num_beliefs_up_for_deletion} beliefs on sensors of {join_words_into_a_list([repr(asset) for asset in generic_assets])}?"
     click.confirm(prompt, abort=True)
-
-    # Delete all beliefs found by query
-    beliefs_up_for_deletion = db.session.scalars(q).all()
-    batch_size = 10000
-    for i, b in enumerate(beliefs_up_for_deletion, start=1):
-        if i % batch_size == 0 or i == num_beliefs_up_for_deletion:
-            click.echo(f"{i} beliefs processed ...")
-        db.session.delete(b)
+    db.session.execute(delete(TimedBelief).where(*entity_filters, *event_filters))
     click.secho(f"Removing {num_beliefs_up_for_deletion} beliefs ...")
     db.session.commit()
     num_beliefs_after = db.session.scalar(select(func.count()).select_from(q))
     # only show the entity names for the final confirmation
+    message = f"{num_beliefs_after} beliefs left on sensors "
     if sensors:
-        done(
-            f"{num_beliefs_after} beliefs left on sensors {join_words_into_a_list([sensor.name for sensor in sensors])}."
-        )
+        message += f"{join_words_into_a_list([sensor.name for sensor in sensors])}"
     elif generic_assets:
-        done(
-            f"{num_beliefs_after} beliefs left on sensors of {join_words_into_a_list([asset.name for asset in generic_assets])}."
+        message += (
+            f"of {join_words_into_a_list([asset.name for asset in generic_assets])}"
         )
+    if start is not None or end is not None:
+        message += " within the specified time window"
+    message += "."
+    done(message)
 
 
 @fm_delete_data.command("unchanged-beliefs", cls=DeprecatedOptionsCommand)

flexmeasures/data/models/generic_assets.py

@@ -137,6 +137,16 @@ def asset_type(self) -> GenericAssetType:
         ),
     )
 
+    @property
+    def offspring(self) -> list[GenericAsset]:
+        """Returns a flattened list of all offspring, which is looked up recursively."""
+        offspring = []
+
+        for child in self.child_assets:
+            offspring.extend(child.offspring)
+
+        return offspring + self.child_assets
+
     @property
     def location(self) -> tuple[float, float] | None:
         location = (self.latitude, self.longitude)

flexmeasures/data/models/planning/tests/conftest.py

@@ -15,7 +15,7 @@
 def app_with_each_solver(app, request):
     """Set up the app config to run with different solvers.
 
-    A test that uses this fixtures runs all of its test cases with HiGHS and then again with Cbc.
+    A test that uses this fixture runs all of its test cases with HiGHS and then again with Cbc.
     """
     original_solver = app.config["FLEXMEASURES_LP_SOLVER"]
     app.config["FLEXMEASURES_LP_SOLVER"] = request.param