Merge branch 'main' into feature/reporting/pandas-reporter-belief-hor…

…izon

.gitignore

@@ -3,7 +3,7 @@ dist
 
 raw_data
 instance
-venv
+*venv/
 .eggs
 .env
 .vscode
@@ -44,3 +44,7 @@ pyproject.toml
 poetry.lock
 .venv
 .vscode/
+
+.DS_Store
+
+.gitconfig.*

.vscode/spellright.dict

@@ -272,3 +272,9 @@ cron
 CSV
 UI
 frontend
+http
+https
+balancer
+url
+HTTPS
+Werkzeug

Makefile

@@ -1,10 +1,11 @@
 # Check Python major and minor version
 # For more information, see https://stackoverflow.com/a/22105036
 PYV = $(shell python -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)")
+HIGHS_DIR = "../HiGHS"
 
 # Note: use tabs
 # actions which are virtual, i.e. not a script
-.PHONY: install install-for-dev install-for-test install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db cli-autocomplete
+.PHONY: install install-for-dev install-for-test install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db cli-autocomplete build-highs-macos install-highs-macos
 
 
 # ---- Development ---
@@ -55,6 +56,29 @@ else
 	rm temp-test.in
 endif
 	make install-flexmeasures
+# Locally install HiGS on macOS
+	if [ "$(shell uname)" = "Darwin" ]; then \
+		make install-highs-macos; \
+	fi
+
+$(HIGHS_DIR):
+	if [ ! -d $(HIGHS_DIR) ]; then \
+		git clone https://github.com/ERGO-Code/HiGHS.git $(HIGHS_DIR); \
+	fi
+	brew install cmake;
+
+build-highs-macos: $(HIGHS_DIR)
+	cd $(HIGHS_DIR); \
+	git checkout latest; \
+	mkdir -p build; \
+	cd build; \
+	cmake ..; \
+	make; \
+	make install; \
+	cd ../../flexmeasures;
+
+install-highs-macos: build-highs-macos
+	pip install $(HIGHS_DIR) ; \
 
 install-deps:
 	make install-pip-tools

ci/update-packages.sh

@@ -29,8 +29,8 @@ if ! [ -x "$(command -v docker)" ]; then
   exit 1
 fi
 
-# Check if we can run docker without sudo
-if ! docker ps > /dev/null 2>&1; then
+# Check if we can run docker without sudo (check is not needed for Macos system)
+if ! docker ps > /dev/null 2>&1 && [[ "$(uname)" != "Darwin" ]]; then
   echo "Docker is not running without sudo. Please add your user to the docker group and try again."
   echo "You may use the following command to do so:"
   echo "sudo usermod -aG docker $USER"

documentation/changelog.rst

@@ -7,6 +7,8 @@ FlexMeasures Changelog
 v0.20.0 | April XX, 2024
 ============================
 
+.. warning:: From this version on, the config setting `FLEXMEASURES_FORCE_HTTPS` decides whether to enforce HTTPS on requests - and it defaults to `False`. Previously, this was governed by `Flask_ENV` or `FLEXMEASURES_ENV` being set to something else than "documentation" or "development". This new way is more clear, but you might be in need of using this setting before upgrading.
+
 New features
 -------------   
 
@@ -18,7 +20,10 @@ New features
 Infrastructure / Support
 ----------------------
 
-
+* Improve processing time for deleting beliefs via CLI [see `PR #1005 <https://github.com/FlexMeasures/flexmeasures/pull/1005>`_]
+* Support deleting beliefs via CLI for all offspring assets at once [see `PR #1003 <https://github.com/FlexMeasures/flexmeasures/pull/1003>`_]
+* Add setting ``FLEXMEASURES_FORCE_HTTPS`` to explicitly toggle if HTTPS should be used for all requests [see `PR #1008 <https://github.com/FlexMeasures/flexmeasures/pull/1008>`_]
+* Make flexmeasures installable locally on macOS [see `PR #1000 <https://github.com/FlexMeasures/flexmeasures/pull/1000>`_]
 
 
 v0.19.2 | March 1, 2024

documentation/configuration.rst

@@ -455,6 +455,14 @@ Allows users to make authenticated requests. If true, injects the Access-Control
 Default: ``True``
 
 
+FLEXMEASURES_FORCE_HTTPS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Set to ``True`` if all requests should be forced to be HTTPS.
+
+Default: ``False``
+
+
 FLEXMEASURES_ENFORCE_SECURE_CONTENT_POLICY
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

documentation/dev/dependency-management.rst

@@ -34,7 +34,7 @@ Now ― you probably have only one Python version installed. Let's say you add a
 
 .. code-block:: bash
 
-    $ cd ci; ./update-packages.sh
+    $ cd ci; ./update-packages.sh; cd ../
 
 This script will use docker to do these upgrades per Python version.
 

documentation/dev/setup-and-guidelines.rst

@@ -54,12 +54,23 @@ Go into the ``flexmeasures`` folder and install all dependencies including the o
 
    $ pip install highspy
 
+On MacOS it will be installed locally by `make install-for-test` and no actions are required on your part
 
-Alternatively, the CBC solver can be installed with:
+Besides highs, the CBC solver is required for tests as well:
 
-.. code-block:: bash
+.. tabs::
+
+    .. tab:: Linux
+
+        .. code-block:: bash
+
+            $ apt-get install coinor-cbc
+
+    .. tab:: MacOS
+
+        .. code-block:: bash
 
-   $ apt-get install coinor-cbc
+            $ brew install cbc
 
 
 Configuration

documentation/host/data.rst

@@ -50,10 +50,23 @@ On Windows:
 * ``conda install psycopg2``
 
 
+On Macos:
+
+.. code-block:: bash
+
+   $ brew update
+   $ brew doctor
+   $ # Need to specify postgres version, in this example we use 13
+   $ brew install postgresql@13
+   $ brew link postgresql@13 --force
+   $ # Start postgres (you can change /usr/local/var/postgres to any directory you like)
+   $ pg_ctl -D /usr/local/var/postgres -l logfile start
+
+
 Using Docker Compose:
 
 
-Alternatively, you can use Docker Compose to run a postgres database. Use can use the following ``docker-compose.yml`` as a starting point:
+Alternatively, you can use Docker Compose to run a postgres database. You can use the following ``docker-compose.yml`` as a starting point:
 
 
 .. code-block:: yaml
@@ -96,9 +109,19 @@ Find the ``timezone`` setting and set it to 'UTC'.
 
 Then restart the postgres server.
 
-.. code-block:: bash
+.. tabs::
+
+   .. tab:: Linux
 
-    $ sudo service postgresql restart
+      .. code-block:: bash
+
+         $ sudo service postgresql restart
+
+   .. tab:: Macos
+
+      .. code-block:: bash
+
+         $ pg_ctl -D /usr/local/var/postgres -l logfile restart
 
 .. note:: If you are using Docker to run postgres, the ``timezone`` setting is already set to ``UTC`` by default.
 
@@ -120,14 +143,16 @@ Proceed to create a database as the postgres superuser (using your postgres user
    $ createuser --pwprompt -U postgres flexmeasures_test  # enter "flexmeasures_test" as password
    $ exit
 
+.. note:: In case you encounter the following "FAILS: sudo: unknown user postgres" you need to create "postgres" OS user with sudo rights first - better done via System preferences -> Users & Groups.
+
 
 Or, from within Postgres console:
 
 .. code-block:: sql
 
-   CREATE USER flexmeasures WITH UNENCRYPTED PASSWORD 'this-is-your-secret-choice';
+   CREATE USER flexmeasures WITH PASSWORD 'this-is-your-secret-choice';
    CREATE DATABASE flexmeasures WITH OWNER = flexmeasures;
-   CREATE USER flexmeasures_test WITH UNENCRYPTED PASSWORD 'flexmeasures_test';
+   CREATE USER flexmeasures_test WITH PASSWORD 'flexmeasures_test';
    CREATE DATABASE flexmeasures_test WITH OWNER = flexmeasures_test;
 
 
@@ -158,6 +183,8 @@ Add the following extensions while logged in as the postgres superuser:
    CREATE EXTENSION cube;
    CREATE EXTENSION earthdistance;
 
+.. note:: Lines from above should be run seperately
+
 
 If you have it, connect to the ``flexmeasures_test`` database and repeat creating these extensions there. Then ``exit``.
 
@@ -230,6 +257,7 @@ You can create users with the ``add user`` command. Check it out:
 
 .. code-block:: bash
 
+   $ flexmeasures add account --help
    $ flexmeasures add user --help
 
 
@@ -281,6 +309,8 @@ You can visualise the data model like this:
 This will generate a picture based on the model code.
 You can also generate picture based on the actual database, see inside the Makefile. 
 
+.. note:: If you encounter "error: externally-managed-environment" when running `make test` and you do it in venv, try `pip cache purge` or use pipx.
+
 Maintenance
 ----------------
 

documentation/tut/toy-example-setup.rst

@@ -52,7 +52,7 @@ Install Flexmeasures and the database
             $ docker run --rm --name flexmeasures-tutorial-db -e POSTGRES_PASSWORD=fm-db-passwd -e POSTGRES_DB=flexmeasures-db -d --network=flexmeasures_network postgres:latest
             $ docker run --rm --name flexmeasures-tutorial-fm --env SQLALCHEMY_DATABASE_URI=postgresql://postgres:fm-db-passwd@flexmeasures-tutorial-db:5432/flexmeasures-db --env SECRET_KEY=notsecret --env FLEXMEASURES_ENV=development --env LOGGING_LEVEL=INFO -d --network=flexmeasures_network -p 5000:5000 lfenergy/flexmeasures
 
-        To upgrade the FlexMeasures database, execute:
+        To establish the FlexMeasures database structure, execute:
 
         .. code-block:: bash
 

flexmeasures/app.py

@@ -106,7 +106,7 @@ def create(  # noqa C901
     set_secret_key(app)
     if app.config.get("SECURITY_PASSWORD_SALT", None) is None:
         app.config["SECURITY_PASSWORD_SALT"] = app.config["SECRET_KEY"]
-    if app.config.get("FLEXMEASURES_ENV") not in ("documentation", "development"):
+    if app.config.get("FLEXMEASURES_FORCE_HTTPS", False):
         SSLify(app)
 
     # Prepare profiling, if needed

flexmeasures/cli/data_delete.py

@@ -256,11 +256,13 @@ def delete_prognoses(
     required=False,
     help="Remove beliefs about events ending at this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.",
 )
+@click.option("--offspring", type=bool, required=False, default=False, is_flag=True)
 def delete_beliefs(  # noqa: C901
     generic_assets: list[GenericAsset],
     sensors: list[Sensor],
     start: datetime | None = None,
     end: datetime | None = None,
+    offspring: bool = False,
 ):
     """Delete all beliefs recorded on a given sensor (or on sensors of a given asset)."""
 
@@ -271,6 +273,8 @@ def delete_beliefs(  # noqa: C901
         abort("Passing both sensors and assets at the same time is not supported.")
     if start is not None and end is not None and start > end:
         abort("Start should not exceed end.")
+    if offspring and len(generic_assets) == 0:
+        abort("Must pass at least one asset when the offspring option is employed.")
 
     # Time window filter
     event_filters = []
@@ -284,6 +288,14 @@ def delete_beliefs(  # noqa: C901
     if sensors:
         entity_filters += [TimedBelief.sensor_id.in_([sensor.id for sensor in sensors])]
     if generic_assets:
+
+        # get the offspring of all generic assets
+        generic_assets_offspring = []
+
+        for asset in generic_assets:
+            generic_assets_offspring.extend(asset.offspring)
+        generic_assets = list(generic_assets) + generic_assets_offspring
+
         entity_filters += [
             TimedBelief.sensor_id == Sensor.id,
             Sensor.generic_asset_id.in_([asset.id for asset in generic_assets]),
@@ -300,26 +312,22 @@ def delete_beliefs(  # noqa: C901
     elif generic_assets:
         prompt = f"Delete all {num_beliefs_up_for_deletion} beliefs on sensors of {join_words_into_a_list([repr(asset) for asset in generic_assets])}?"
     click.confirm(prompt, abort=True)
-
-    # Delete all beliefs found by query
-    beliefs_up_for_deletion = db.session.scalars(q).all()
-    batch_size = 10000
-    for i, b in enumerate(beliefs_up_for_deletion, start=1):
-        if i % batch_size == 0 or i == num_beliefs_up_for_deletion:
-            click.echo(f"{i} beliefs processed ...")
-        db.session.delete(b)
+    db.session.execute(delete(TimedBelief).where(*entity_filters, *event_filters))
     click.secho(f"Removing {num_beliefs_up_for_deletion} beliefs ...")
     db.session.commit()
     num_beliefs_after = db.session.scalar(select(func.count()).select_from(q))
     # only show the entity names for the final confirmation
+    message = f"{num_beliefs_after} beliefs left on sensors "
     if sensors:
-        done(
-            f"{num_beliefs_after} beliefs left on sensors {join_words_into_a_list([sensor.name for sensor in sensors])}."
-        )
+        message += f"{join_words_into_a_list([sensor.name for sensor in sensors])}"
     elif generic_assets:
-        done(
-            f"{num_beliefs_after} beliefs left on sensors of {join_words_into_a_list([asset.name for asset in generic_assets])}."
+        message += (
+            f"of {join_words_into_a_list([asset.name for asset in generic_assets])}"
         )
+    if start is not None or end is not None:
+        message += " within the specified time window"
+    message += "."
+    done(message)
 
 
 @fm_delete_data.command("unchanged-beliefs", cls=DeprecatedOptionsCommand)

flexmeasures/data/models/generic_assets.py

@@ -137,6 +137,16 @@ def asset_type(self) -> GenericAssetType:
         ),
     )
 
+    @property
+    def offspring(self) -> list[GenericAsset]:
+        """Returns a flattened list of all offspring, which is looked up recursively."""
+        offspring = []
+
+        for child in self.child_assets:
+            offspring.extend(child.offspring)
+
+        return offspring + self.child_assets
+
     @property
     def location(self) -> tuple[float, float] | None:
         location = (self.latitude, self.longitude)

flexmeasures/data/models/planning/tests/conftest.py

@@ -15,7 +15,7 @@
 def app_with_each_solver(app, request):
     """Set up the app config to run with different solvers.
 
-    A test that uses this fixtures runs all of its test cases with HiGHS and then again with Cbc.
+    A test that uses this fixture runs all of its test cases with HiGHS and then again with Cbc.
     """
     original_solver = app.config["FLEXMEASURES_LP_SOLVER"]
     app.config["FLEXMEASURES_LP_SOLVER"] = request.param

flexmeasures/data/scripts/clean_database.sh

@@ -100,14 +100,14 @@ if is_database $1
 then
   echo "$1 database exists"
   read -r -p "Make a backup first? [y/N] " response
-  response=${response,,}    # make lowercase
+  response=$(tr '[:upper:]' '[:lower:]' <<< $response) # make lowercase
   if [[ "$response" =~ ^(yes|y)$ ]]; then
     echo "Making db dump ..."
     flexmeasures db-ops dump
   fi
 
   read -r -p "This will drop your database and re-create a clean one. Continue?[y/N] " response
-  response=${response,,} # make lowercase
+  response=$(tr '[:upper:]' '[:lower:]' <<< $response) # make lowercase
   if [[ "$response" =~ ^(yes|y)$ ]]; then
      if ! delete_database $1; then
        exit 1