Merge branch 'main' into 433-fetch-one

.github/workflows/lint-and-test.yml

@@ -1,7 +1,10 @@
 name: lint-and-test
 
-on: ["push", "pull_request"]
-
+on:
+  push:
+  pull_request:
+    types:
+      - opened
 jobs:
   check:
     runs-on: ubuntu-latest
@@ -35,7 +38,8 @@ jobs:
         run: |
           git fetch --prune --unshallow
           git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-      - uses: actions/cache@v2
+      - name: "Caching dependencies (txt)"
+        uses: actions/cache@v2
         id: cache
         with:
           path: ${{ env.pythonLocation }}
@@ -46,21 +50,16 @@ jobs:
       - run: |
           ci/setup-postgres.sh
           sudo apt-get -y install coinor-cbc
-      - name: Install FlexMeasures & dependencies for tests
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          make install-pip-tools
-          make install-for-test
-          pip install coveralls
-      - name: Run all tests except those marked to be skipped by GitHub
-        run: pytest -m "not skip_github"
-        if: ${{ matrix.coverage != 'yes' }}
+      - name: Install FlexMeasures & exact dependencies for tests
+        run: make install-for-test
+        if: github.event_name == 'push' && steps.cache.outputs.cache-hit != 'true'
+      - name: Install FlexMeasures & latest dependencies for tests
+        run: make install-for-test pinned=no
+        if: github.event_name == 'pull_request'
       - name: Run all tests except those marked to be skipped by GitHub AND record coverage
-        run: pytest -v -m "not skip_github" --cov=flexmeasures --cov-branch
-        if: ${{ matrix.coverage == 'yes' }}
-      - run: coveralls --service=github
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: pytest -v -m "not skip_github" --cov=flexmeasures --cov-branch --cov-report=lcov
+      - name: Coveralls
+        uses: coverallsapp/github-action@v2
         if: ${{ matrix.coverage == 'yes' }}
     env:
       PGHOST: 127.0.0.1

.vscode/settings.json

@@ -13,5 +13,10 @@
     "python.linting.pylintEnabled": false,
     "python.linting.flake8Enabled": true,
     "workbench.editor.wrapTabs": true,
-    "python.formatting.provider": "black"
+    "python.formatting.provider": "black",
+    "python.testing.pytestArgs": [
+        "flexmeasures"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
 }

Makefile

@@ -1,6 +1,6 @@
 # Note: use tabs
 # actions which are virtual, i.e. not a script
-.PHONY: install install-for-dev install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db
+.PHONY: install install-for-dev install-for-test install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db
 
 
 # ---- Development ---
@@ -39,13 +39,27 @@ install-for-dev:
 	make install-flexmeasures
 
 install-for-test:
-	pip-sync requirements/app.txt requirements/dev.txt requirements/test.txt
+	make install-pip-tools
+# Pass pinned=no if you want to test against latest stable packages, default is our pinned dependency set
+ifneq ($(pinned), no)
+	pip-sync requirements/app.txt requirements/test.txt
+else
+	# cutting off the -c inter-layer dependency (that's pip-tools specific)
+	tail -n +3 requirements/test.in >> temp-test.in
+	pip install --upgrade -r requirements/app.in -r temp-test.in
+	rm temp-test.in
+endif
 	make install-flexmeasures
 
 install-deps:
 	make install-pip-tools
 	make freeze-deps
+# Pass pinned=no if you want to test against latest stable packages, default is our pinned dependency set
+ifneq ($(pinned), no)
 	pip-sync requirements/app.txt
+else
+	pip install --upgrade -r requirements/app.in
+endif
 
 install-flexmeasures:
 	pip install -e .

documentation/api/change_log.rst

@@ -9,6 +9,7 @@ v3.0-10 | 2023-06-12
 """"""""""""""""""""
 
 - Introduced the ``storage-efficiency`` field to the ``flex-model``field for `/sensors/<id>/schedules/trigger` (POST).
+- Introduced the ``database_redis`` optional field to the response of the endpoint `/health/ready` (GET).
 
 v3.0-9 | 2023-04-26
 """""""""""""""""""

documentation/changelog.rst

@@ -6,12 +6,15 @@ FlexMeasures Changelog
 v0.15.0 | July XX, 2023
 ============================
 
+.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``).
+
 New features
 -------------
 
 * Allow deleting multiple sensors with a single call to ``flexmeasures delete sensor`` by passing the ``--id`` option multiple times [see `PR #734 <https://www.github.com/FlexMeasures/flexmeasures/pull/734>`_]
 * Make it a lot easier to read off the color legend on the asset page, especially when showing many sensors, as they will now be ordered from top to bottom in the same order as they appear in the chart (as defined in the ``sensors_to_show`` attribute), rather than alphabetically [see `PR #742 <https://www.github.com/FlexMeasures/flexmeasures/pull/742>`_]
 * Having percentages within the [0, 100] domain is such a common use case that we now always include it in sensor charts with % units, making it easier to read off individual charts and also to compare across charts [see `PR #739 <https://www.github.com/FlexMeasures/flexmeasures/pull/739>`_]
+* DataSource table now allows storing arbitrary attributes as a JSON (without content validation), similar to the Sensor and GenericAsset tables [see `PR #750 <https://www.github.com/FlexMeasures/flexmeasures/pull/750>`_]
 
 Bugfixes
 -----------
@@ -20,7 +23,10 @@ Infrastructure / Support
 ----------------------
 
 * Add support for profiling Flask API calls using ``pyinstrument`` (if installed). Can be enabled by setting the environment variable ``FLEXMEASURES_PROFILE_REQUESTS`` to ``True`` [see `PR #722 <https://www.github.com/FlexMeasures/flexmeasures/pull/722>`_]
+* The endpoint `[POST] /health/ready <api/v3_0.html#get--api-v3_0-health-ready>`_ returns the status of the Redis connection, if configured [see `PR #699 <https://www.github.com/FlexMeasures/flexmeasures/pull/699>`_]
+
 
+/api/v3_0/health/ready
 
 v0.14.1 | June 26, 2023
 ============================

documentation/conf.py

@@ -221,7 +221,7 @@
 # -- Options for intersphinx extension ---------------------------------------
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {"https://docs.python.org/3/": None}
+intersphinx_mapping = {"python": ("https://docs.python.org/3", None)}
 
 # -- Options for copybytton extension ---------------------------------------
 copybutton_prompt_is_regexp = True

documentation/index.rst

@@ -180,6 +180,7 @@ The platform operator of FlexMeasures can be an Aggregator.
     :maxdepth: 1
 
     tut/toy-example-from-scratch
+    tut/toy-example-expanded
     tut/installation
     tut/posting_data
     tut/forecasting_scheduling

documentation/tut/installation.rst

@@ -3,8 +3,8 @@
 Installation & First steps
 =================================
 
-Getting FlexMeasures to run
------------------------------
+Preparing FlexMeasures for running
+------------------------------------
 
 This section walks you through installing FlexMeasures on your own PC and running it continuously.
 We'll cover getting started by making a secret key, connecting a database and creating one user & one asset.
@@ -194,12 +194,8 @@ Finally, you can tell FlexMeasures to create forecasts for your meter data with
 .. note:: You can also use the API to send forecast data.
 
 
-Run FlexMeasures
-------------------
-
-
-Running the web service
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+Running FlexMeasures as a web service
+--------------------------------------
 
 It's finally time to start running FlexMeasures:
 

documentation/tut/posting_data.rst

@@ -94,11 +94,11 @@ If this resolution does not match the sensor's resolution, FlexMeasures will try
 Likewise, if the data unit does not match the sensor’s unit, FlexMeasures will attempt to convert the data or, if that is not possible, complain.
 
 
-Posting power data
-------------------
+Being explicit when posting power data
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 For power data, USEF specifies separate message types for observations and forecasts.
-Correspondingly, we allow the following message types to be used with the [POST] /sensors/data endpoint (see :ref:`posting_sensor_data`):
+Correspondingly, we allow the following message types to be used with the `POST  /sensors/data <../api/v3_0.html#post--api-v3_0-sensors-data>`_ endpoint:
 
 .. code-block:: json
 
@@ -239,8 +239,8 @@ Multiple values (indicating a univariate timeseries) for 15-minute time interval
 
 .. _observations_vs_forecasts
 
-Observations vs forecasts
---------------------------
+Observations vs forecasts: The time of knowledge
+-------------------------------------------------
 
 To correctly tell FlexMeasures when a meter reading or forecast was known is crucial, as it determines which data is being used to compute schedules or to make other forecasts.
 

documentation/tut/toy-example-expanded.rst

@@ -0,0 +1,106 @@
+.. _tut_toy_schedule_expanded:
+
+
+
+Toy example II: Adding solar production and limited grid connection
+====================================================================
+
+
+So far we haven't taken into account any other devices that consume or produce electricity. The battery was free to use all available capacity towards the grid. 
+
+What if other devices will be using some of that capacity? Our schedules need to reflect that, so we stay within given limits.
+
+.. note:: The capacity is given by ``capacity_in_mw``, an attribute we placed on the battery asset earlier (see :ref:`tut_toy_schedule`). We will tell FlexMeasures to take the solar production into account (using ``--inflexible-device-sensor``) for this capacity limit.
+
+We'll now add solar production forecast data and then ask for a new schedule, to see the effect of solar on the available headroom for the battery.
+
+
+Adding PV production forecasts
+------------------------------
+
+First, we'll create a new CSV file with solar forecasts (MW, see the setup for sensor 3 in part I of this tutorial) for tomorrow.
+
+.. code-block:: bash
+
+    $ TOMORROW=$(date --date="next day" '+%Y-%m-%d')
+    $ echo "Hour,Price
+    $ ${TOMORROW}T00:00:00,0.0
+    $ ${TOMORROW}T01:00:00,0.0
+    $ ${TOMORROW}T02:00:00,0.0
+    $ ${TOMORROW}T03:00:00,0.0
+    $ ${TOMORROW}T04:00:00,0.01
+    $ ${TOMORROW}T05:00:00,0.03
+    $ ${TOMORROW}T06:00:00,0.06
+    $ ${TOMORROW}T07:00:00,0.1
+    $ ${TOMORROW}T08:00:00,0.14
+    $ ${TOMORROW}T09:00:00,0.17
+    $ ${TOMORROW}T10:00:00,0.19
+    $ ${TOMORROW}T11:00:00,0.21
+    $ ${TOMORROW}T12:00:00,0.22
+    $ ${TOMORROW}T13:00:00,0.21
+    $ ${TOMORROW}T14:00:00,0.19
+    $ ${TOMORROW}T15:00:00,0.17
+    $ ${TOMORROW}T16:00:00,0.14
+    $ ${TOMORROW}T17:00:00,0.1
+    $ ${TOMORROW}T18:00:00,0.06
+    $ ${TOMORROW}T19:00:00,0.03
+    $ ${TOMORROW}T20:00:00,0.01
+    $ ${TOMORROW}T21:00:00,0.0
+    $ ${TOMORROW}T22:00:00,0.0
+    $ ${TOMORROW}T23:00:00,0.0" > solar-tomorrow.csv
+
+Then, we read in the created CSV file as beliefs data.
+This time, different to above, we want to use a new data source (not the user) ― it represents whoever is making these solar production forecasts.
+We create that data source first, so we can tell `flexmeasures add beliefs` to use it.
+Setting the data source type to "forecaster" helps FlexMeasures to visually distinguish its data from e.g. schedules and measurements.
+
+.. note:: The ``flexmeasures add source`` command also allows to set a model and version, so sources can be distinguished in more detail. But that is not the point of this tutorial. See ``flexmeasures add source --help``.
+
+.. code-block:: bash
+
+    $ flexmeasures add source --name "toy-forecaster" --type forecaster
+    Added source <Data source 4 (toy-forecaster)>
+    $ flexmeasures add beliefs --sensor-id 3 --source 4 solar-tomorrow.csv --timezone Europe/Amsterdam
+    Successfully created beliefs
+
+The one-hour CSV data is automatically resampled to the 15-minute resolution of the sensor that is recording solar production. We can see solar production in the `FlexMeasures UI <http://localhost:5000/sensors/3/>`_ :
+
+.. image:: https://github.com/FlexMeasures/screenshots/raw/main/tut/toy-schedule/sensor-data-production.png
+    :align: center
+|
+
+.. note:: The ``flexmeasures add beliefs`` command has many options to make sure the read-in data is correctly interpreted (unit, timezone, delimiter, etc). But that is not the point of this tutorial. See ``flexmeasures add beliefs --help``.
+
+
+Trigger an updated schedule
+----------------------------
+
+Now, we'll reschedule the battery while taking into account the solar production. This will have an effect on the available headroom for the battery, given the ``capacity_in_mw`` limit discussed earlier.
+
+.. code-block:: bash
+
+    $ flexmeasures add schedule for-storage --sensor-id 1 --consumption-price-sensor 2 \
+        --inflexible-device-sensor 3 \
+        --start ${TOMORROW}T07:00+01:00 --duration PT12H \
+        --soc-at-start 50% --roundtrip-efficiency 90%
+    New schedule is stored.
+
+We can see the updated scheduling in the `FlexMeasures UI <http://localhost:5000/sensors/1/>`_ :
+
+.. image:: https://github.com/FlexMeasures/screenshots/raw/main/tut/toy-schedule/sensor-data-charging-with-solar.png
+    :align: center
+|
+
+The `asset page for the battery <http://localhost:5000/assets/1/>`_ now shows the solar data, too:
+
+.. image:: https://github.com/FlexMeasures/screenshots/raw/main/tut/toy-schedule/asset-view-with-solar.png
+    :align: center
+
+
+Though this schedule is quite similar, we can see that it has changed from `the one we computed earlier <https://raw.githubusercontent.com/FlexMeasures/screenshots/main/tut/toy-schedule/asset-view-without-solar.png>`_ (when we did not take solar into account).
+
+First, during the sunny hours of the day, when solar power is being send to the grid, the battery's output (at around 9am and 11am) is now lower, as the battery shares ``capacity_in_mw`` with the solar production. In the evening (around 7pm), when solar power is basically not present anymore, battery discharging to the grid is still at its previous levels.
+
+Second, charging of the battery is also changed a bit (around 10am), as less can be discharged later.
+
+We hope this part of the tutorial shows how to incorporate a limited grid connection rather easily with FlexMeasures. There are more ways to model such settings, but this is a straightforward one.