Merge branch 'main' into 632-make-pip-installable-on-windows

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,2 +1,27 @@
+## Description
+
+Summary of the changes introduced in this PR. Try to use bullet points as much as possible.
+
+## Look & Feel
+
+This section can contain example pictures for UI, Input/Output for CLI, Request / Response for API endpoint, etc.
+
+## How to test
+
+Steps to test it or name of the tests functions.
+
+The library [flexmeasures-client](https://github.com/FlexMeasures/flexmeasures-client/) can be useful to showcase new features. For example,
+it can be used to set some example data to be used in a new UI feature.
+
+## Further Improvements
+
+Potential improvements to be done in the same PR or follow up Issues/Discussions/PRs.
+
+## Related Items
+
+Mention if this PR closes an Issue or Project.
+
+---
+
 - [ ] I agree to contribute to the project under Apache 2 License. 
-- [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or other license that is incompatible with FlexMeasures
+- [ ] To the best of my knowledge, the proposed patch is not based on code under GPL or other license that is incompatible with FlexMeasures

.github/workflows/lint-and-test.yml

@@ -1,7 +1,10 @@
 name: lint-and-test
 
-on: ["push", "pull_request"]
-
+on:
+  push:
+  pull_request:
+    types:
+      - opened
 jobs:
   check:
     runs-on: ubuntu-latest
@@ -35,7 +38,8 @@ jobs:
         run: |
           git fetch --prune --unshallow
           git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-      - uses: actions/cache@v2
+      - name: "Caching dependencies (txt)"
+        uses: actions/cache@v2
         id: cache
         with:
           path: ${{ env.pythonLocation }}
@@ -46,21 +50,16 @@ jobs:
       - run: |
           ci/setup-postgres.sh
           sudo apt-get -y install coinor-cbc
-      - name: Install FlexMeasures & dependencies for tests
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          make install-pip-tools
-          make install-for-test
-          pip install coveralls
-      - name: Run all tests except those marked to be skipped by GitHub
-        run: pytest -m "not skip_github"
-        if: ${{ matrix.coverage != 'yes' }}
+      - name: Install FlexMeasures & exact dependencies for tests
+        run: make install-for-test
+        if: github.event_name == 'push' && steps.cache.outputs.cache-hit != 'true'
+      - name: Install FlexMeasures & latest dependencies for tests
+        run: make install-for-test pinned=no
+        if: github.event_name == 'pull_request'
       - name: Run all tests except those marked to be skipped by GitHub AND record coverage
-        run: pytest -v -m "not skip_github" --cov=flexmeasures --cov-branch
-        if: ${{ matrix.coverage == 'yes' }}
-      - run: coveralls --service=github
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: pytest -v -m "not skip_github" --cov=flexmeasures --cov-branch --cov-report=lcov
+      - name: Coveralls
+        uses: coverallsapp/github-action@v2
         if: ${{ matrix.coverage == 'yes' }}
     env:
       PGHOST: 127.0.0.1

.gitignore

@@ -29,10 +29,13 @@ notebooks/.ipynb_checkpoints/
 
 flexmeasures/ui/static/documentation
 documentation/img/screenshot_*
+documentation/_autosummary/
 generic_asset_fm_user_ownership.sql
 
 uml_diagram.png
 db_schema.png
 
 .coverage
 htmlcov
+test/*
+profile_reports/*

.vscode/settings.json

@@ -13,5 +13,10 @@
     "python.linting.pylintEnabled": false,
     "python.linting.flake8Enabled": true,
     "workbench.editor.wrapTabs": true,
-    "python.formatting.provider": "black"
+    "python.formatting.provider": "black",
+    "python.testing.pytestArgs": [
+        "flexmeasures"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
 }

.vscode/spellright.dict

@@ -251,3 +251,9 @@ Changelog
 Bugfixes
 Dockerfile
 nt
+Backoffice
+eval
+dataframe
+dataframes
+args
+docstrings

Makefile

@@ -1,6 +1,6 @@
 # Note: use tabs
 # actions which are virtual, i.e. not a script
-.PHONY: install install-for-dev install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db
+.PHONY: install install-for-dev install-for-test install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db
 
 
 # ---- Development ---
@@ -14,18 +14,20 @@ test:
 
 # ---- Documentation ---
 
+gen_code_docs := False # by default code documentation is not generated
+
 update-docs:
 	@echo "Creating docs environment ..."
 	make install-docs-dependencies
 	@echo "Creating documentation ..."
-	cd documentation; make clean; make html SPHINXOPTS="-W --keep-going -n"; cd ..
+	export GEN_CODE_DOCS=${gen_code_docs}; cd documentation; make clean; make html SPHINXOPTS="-W --keep-going -n"; cd ..
 
 update-docs-pdf:
 	@echo "NOTE: PDF documentation requires packages (on Debian: latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended)"
 	@echo "NOTE: Currently, the docs require some pictures which are not in the git repo atm. Ask the devs."
 	make install-sphinx-tools
 
-	cd documentation; make clean; make latexpdf; make latexpdf; cd ..  # make latexpdf can require two passes
+	export GEN_CODE_DOCS=${gen_code_docs}; cd documentation; make clean; make latexpdf; make latexpdf; cd ..  # make latexpdf can require two passes
 
 # ---- Installation ---
 
@@ -37,13 +39,27 @@ install-for-dev:
 	make install-flexmeasures
 
 install-for-test:
-	pip-sync requirements/app.txt requirements/dev.txt requirements/test.txt
+	make install-pip-tools
+# Pass pinned=no if you want to test against latest stable packages, default is our pinned dependency set
+ifneq ($(pinned), no)
+	pip-sync requirements/app.txt requirements/test.txt
+else
+	# cutting off the -c inter-layer dependency (that's pip-tools specific)
+	tail -n +3 requirements/test.in >> temp-test.in
+	pip install --upgrade -r requirements/app.in -r temp-test.in
+	rm temp-test.in
+endif
 	make install-flexmeasures
 
 install-deps:
 	make install-pip-tools
 	make freeze-deps
+# Pass pinned=no if you want to test against latest stable packages, default is our pinned dependency set
+ifneq ($(pinned), no)
 	pip-sync requirements/app.txt
+else
+	pip install --upgrade -r requirements/app.in
+endif
 
 install-flexmeasures:
 	pip install -e .

documentation/_templates/custom-module-template.rst

@@ -0,0 +1,67 @@
+.. Adapted from https://stackoverflow.com/a/62613202
+{{ fullname | escape | underline}}
+
+{% block modules %}
+{% if modules %}
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+   :template: custom-module-template.rst                
+   :recursive:
+{% for item in modules %}
+   {% if "test" not in item %}
+   {{ item }}
+   {% endif %}
+{%- endfor %}
+{% endif %}
+{% endblock %}
+
+.. automodule:: {{ fullname }}
+
+   {% block attributes %}
+   {% if attributes %}
+   .. rubric:: Module Attributes
+
+
+   {% for item in attributes %}
+   .. autoattribute::
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block functions %}
+   {% if functions %}
+   .. rubric:: {{ _('Functions') }}
+
+   {% for item in functions %}
+   .. autofunction::
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block classes %}
+   {% if classes %}
+   .. rubric:: {{ _('Classes') }}
+
+   {% for item in classes %}     
+   .. autoclass:: {{ item }}
+      :members:
+      :special-members: __init__
+      :private-members:
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+
+   {% block exceptions %}
+   {% if exceptions %}
+   .. rubric:: {{ _('Exceptions') }}
+
+   {% for item in exceptions %}
+   .. autoexception::
+      {{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}

documentation/api/change_log.rst

@@ -5,6 +5,17 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace.
 
+v3.0-11 | 2023-07-20
+""""""""""""""""""""
+
+- Added REST endpoint for fetching one sensor: `/sensors/<id>` (GET)
+
+v3.0-10 | 2023-06-12
+""""""""""""""""""""
+
+- Introduced the ``storage-efficiency`` field to the ``flex-model``field for `/sensors/<id>/schedules/trigger` (POST).
+- Introduced the ``database_redis`` optional field to the response of the endpoint `/health/ready` (GET).
+
 v3.0-9 | 2023-04-26
 """""""""""""""""""
 

documentation/api/introduction.rst

@@ -45,7 +45,7 @@ Let's see what the ``/api`` endpoint returns:
     {'flexmeasures_version': '0.9.0',
      'message': 'For these API versions endpoints are available. An authentication token can be requested at: /api/requestAuthToken. For a list of services, see https://flexmeasures.readthedocs.io',
      'status': 200,
-     'versions': ['v1', 'v1_1', 'v1_2', 'v1_3', 'v2_0', 'v3_0']
+     'versions': ['v3_0']
     }
 
 So this tells us which API versions exist. For instance, we know that the latest API version is available at:
@@ -57,6 +57,9 @@ So this tells us which API versions exist. For instance, we know that the latest
 
 Also, we can see that a list of endpoints is available on https://flexmeasures.readthedocs.io for each of these versions.
 
+.. note:: Sunset API versions are still documented there, simply select an older version.
+
+
 .. _api_auth:
 
 Authentication
@@ -111,6 +114,8 @@ When an API feature becomes obsolete, we deprecate it.
 Deprecation of major features doesn't happen a lot, but when it does, it happens in multiple stages, during which we support clients and hosts in adapting.
 For more information on our multi-stage deprecation approach and available options for FlexMeasures hosts, see :ref:`Deprecation and sunset for hosts<api_deprecation_hosts>`.
 
+.. _api_deprecation_clients:
+
 Clients
 ^^^^^^^
 
@@ -154,10 +159,9 @@ Hosts
 
 FlexMeasures versions go through the following stages for deprecating major features (such as API versions):
 
-- :ref:`api_deprecation_stage_1`: status 200 (OK) with relevant headers, plus a toggle to 410 (Gone) for blackout tests
+- :ref:`api_deprecation_stage_1`: status 200 (OK) with :ref:`relevant headers<api_deprecation_clients>`, plus a toggle to 410 (Gone) for blackout tests
 - :ref:`api_deprecation_stage_2`: status 410 (Gone), plus a toggle to 200 (OK) for sunset rollbacks
-- :ref:`api_deprecation_stage_3`: status 404 (Not Found), plus a toggle to 410 (Gone) for removal rollbacks
-- :ref:`api_deprecation_stage_4`: status 404 (Not Found), and removal of relevant endpoints
+- :ref:`api_deprecation_stage_3`: status 410 (Gone)
 
 Let's go over these stages in more detail.
 
@@ -202,15 +206,4 @@ To enable this, just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = F
 Stage 3: Definitive sunset
 """"""""""""""""""""""""""
 
-After upgrading to one of the next FlexMeasures versions (e.g. ``flexmeasures==0.14``), clients that call sunset endpoints will receive ``HTTP status 404 (Not Found)`` responses.
-In case you need clients to receive the slightly more informative ``HTTP status 410 (Gone)``  for a little while longer, we will continue to support a "removal rollback".
-To enable this, just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = True``.
-This, just like in deprecation stages 1 and 2, leads to status 410 (Gone) responses.
-Note that ``FLEXMEASURES_API_SUNSET_ACTIVE = False`` now leads to status 404 (Not Found) responses, unlike in deprecation stages 1 and 2, where this would have lead to status 200 (OK) responses.
-
-.. _api_deprecation_stage_4:
-
-Stage 4: Removal
-""""""""""""""""
-
-After upgrading to one of the next FlexMeasures versions (e.g. ``flexmeasures==0.15``), clients that call sunset endpoints will receive ``HTTP status 404 (Not Found)`` responses.
+After upgrading to one of the next FlexMeasures versions (e.g. ``flexmeasures==0.14``), clients that call sunset endpoints will receive ``HTTP status 410 (Gone)`` responses.

documentation/api/notation.rst

@@ -145,10 +145,8 @@ Technically, this is equal to:
 
 This intuitive convention allows us to reduce communication by sending univariate timeseries as arrays.
 
-Notation for v1, v2 and v3
-""""""""""""""""""""""""""
 
-For version 1, 2 and 3 of the API, only equidistant timeseries data is expected to be communicated. Therefore:
+In all current versions of the FlexMeasures API, only equidistant timeseries data is expected to be communicated. Therefore:
 
 - only the array notation should be used (first notation from above),
 - "start" should be a timestamp on the hour or a multiple of the sensor resolution thereafter (e.g. "16:10" works if the resolution is 5 minutes), and
@@ -185,27 +183,37 @@ This means that API and CLI users don't have to send the whole flex model every
 
 Here are the three types of flexibility models you can expect to be built-in:
 
-1) For storage devices (e.g. batteries, charge points, electric vehicle batteries connected to charge points), the schedule deals with the state of charge (SOC).
+1) For **storage devices** (e.g. batteries, and :abbr:`EV (electric vehicle)` batteries connected to charge points), the schedule deals with the state of charge (SOC).
 
-    The possible flexibility parameters are:
+   The possible flexibility parameters are:
 
-    - ``soc-at-start`` (defaults to 0)
-    - ``soc-unit`` (kWh or MWh)
-    - ``soc-min`` (defaults to 0)
-    - ``soc-max`` (defaults to max soc target)
-    - ``soc-targets`` (defaults to NaN values)
-    - ``roundtrip-efficiency`` (defaults to 100%)
-    - ``prefer-charging-sooner`` (defaults to True, also signals a preference to discharge later)
+   - ``soc-at-start`` (defaults to 0)
+   - ``soc-unit`` (kWh or MWh)
+   - ``soc-min`` (defaults to 0)
+   - ``soc-max`` (defaults to max soc target)
+   - ``soc-minima`` (defaults to NaN values)
+   - ``soc-maxima`` (defaults to NaN values)
+   - ``soc-targets`` (defaults to NaN values)
+   - ``roundtrip-efficiency`` (defaults to 100%)
+   - ``storage-efficiency`` (defaults to 100%) [#]_
+   - ``prefer-charging-sooner`` (defaults to True, also signals a preference to discharge later)
 
-  For some examples, see the `[POST] /sensors/(id)/schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_ endpoint docs.
+    .. [#] The storage efficiency (e.g. 95% or 0.95) to use for the schedule is applied over each time step equal to the sensor resolution. For example, a storage efficiency of 95 percent per (absolute) day, for scheduling a 1-hour resolution sensor, should be passed as a storage efficiency of :math:`0.95^{1/24} = 0.997865`.
 
-2) Shiftable process
-
-   .. todo:: A simple algorithm exists, needs integration into FlexMeasures and asset type clarified.
+   For some examples, see the `[POST] /sensors/(id)/schedules/trigger <../api/v3_0.html#post--api-v3_0-sensors-(id)-schedules-trigger>`_ endpoint docs.
 
-3) Heat pumps
+2) For **shiftable processes**
 
-   .. todo:: Also work in progress, needs model for heat loss compensation.
+   .. todo:: A simple and proven algorithm exists, but is awaiting proper integration into FlexMeasures, see `PR 729 <https://github.com/FlexMeasures/flexmeasures/pull/729>`_.
+
+3) For **buffer devices** (e.g. thermal energy storage systems connected to heat pumps), use the same flexibility parameters described above for storage devices. Here are some tips to model a buffer with these parameters:
+
+   - Describe the thermal energy content in kWh or MWh.
+   - Set ``soc-minima`` to the accumulative usage forecast.
+   - Set ``roundtrip-efficiency`` to the square of the conversion efficiency. [#]_
+   - Set ``storage-efficiency`` to a value below 100% to model (heat) loss.
+
+    .. [#] Setting a roundtrip efficiency of higher than 1 is not supported. We plan to implement a separate field for :abbr:`COP (coefficient of performance)` values.
 
 In addition, folks who write their own custom scheduler (see :ref:`plugin_customization`) might also require their custom flexibility model.
 That's no problem, FlexMeasures will let the scheduler decide which flexibility model is relevant and how it should be validated.