Merge branch 'main' into coveralls-new-gh-action

FlexMeasures · May 25, 2023 · 0e7c7f7 · 0e7c7f7
2 parents 86f2ad8 + fe4c3a9
commit 0e7c7f7
Show file tree

Hide file tree

Showing 145 changed files with 2,827 additions and 7,054 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -1,2 +1,2 @@
 [run]
-omit = */tests/*, */scripts/*
+omit = */tests/*, */scripts/*, **/*.jinja, **/*.html, **/*.txt
diff --git a/.gitignore b/.gitignore
@@ -29,6 +29,7 @@ notebooks/.ipynb_checkpoints/
 
 flexmeasures/ui/static/documentation
 documentation/img/screenshot_*
+generic_asset_fm_user_ownership.sql
 
 uml_diagram.png
 db_schema.png

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -15,13 +15,15 @@ build:
   os: ubuntu-20.04
   tools:
     python: "3.9"
+  jobs:
+    post_create_environment:
+      - pip install . --no-deps  # as python install step, RTD installs deps eagerly
 
 python: 
   install:
     - requirements: requirements/app.txt
     - requirements: requirements/docs.txt
-    - method: pip
-      path: .
+
 
 sphinx:
   fail_on_warning: true
diff --git a/.vscode/spellright.dict b/.vscode/spellright.dict
@@ -246,3 +246,8 @@ deserializing
 kwargs
 fsdomain
 filepath
+spellright
+Changelog
+Bugfixes
+Dockerfile
+nt
diff --git a/Makefile b/Makefile
@@ -46,7 +46,7 @@ install-deps:
 	pip-sync requirements/app.txt
 
 install-flexmeasures:
-	python setup.py develop
+	pip install -e .
 
 install-pip-tools:
 	pip3 install -q "pip-tools>=6.4"
@@ -89,4 +89,4 @@ show-data-model:
 	./flexmeasures/data/scripts/visualize_data_model.py --uml
 
 clean-db:
-	./flexmeasures/data/scripts/clean_database.sh ${db_name}
+	./flexmeasures/data/scripts/clean_database.sh ${db_name} ${db_user}
diff --git a/documentation/_static/css/custom.css b/documentation/_static/css/custom.css
@@ -1,3 +1,10 @@
+div section {
+    text-align: justify;
+}
+#table-of-contents {
+    text-align: left;
+}
+
 div .toctree-wrapper > ul {
     column-count: 2;
     margin: 0;
@@ -20,4 +27,8 @@ div .contents li {
     -webkit-column-break-inside: avoid;
     page-break-inside: avoid;
     break-inside: avoid-column;
-}
+}
+
+div.admonition.info-icon > .admonition-title:before {
+  content: "\f05a"; /* the fa-circle-info icon */
+}
diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst
@@ -5,6 +5,12 @@ API change log
 
 .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace.
 
+v3.0-9 | 2023-04-26
+"""""""""""""""""""
+
+- Added missing documentation for the public endpoints for authentication and listing active API versions.
+- Added REST endpoint for listing available services for a specific API version: `/api/v3_0` (GET). This functionality is similar to the *getService* endpoint for older API versions, but now also returns the full URL for each available service.
+
 v3.0-8 | 2023-03-23
 """""""""""""""""""
 
@@ -82,7 +88,7 @@ v3.0-0 | 2022-03-25
     - *getDeviceMessage* -> use `/sensors/<id>/schedules/<uuid>` (GET) instead, where <id> is the sensor id from the "event" field and <uuid> is the value of the "schedule" field returned by `/sensors/<id>/schedules/trigger` (POST)
     - *getMeterData* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor"
     - *getPrognosis* -> use `/sensors/data` (GET) instead, replacing the "connection" field with "sensor"
-    - *getService* -> consult the public API documentation instead, at https://flexmeasures.readthedocs.io
+    - *getService* -> use `/api/v3_0` (GET) instead (since v3.0-9), or consult the public API documentation instead, at https://flexmeasures.readthedocs.io
     - *postMeterData* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor"
     - *postPriceData* -> use `/sensors/data` (POST) instead, replacing the "market" field with "sensor"
     - *postPrognosis* -> use `/sensors/data` (POST) instead, replacing the "connection" field with "sensor"
@@ -106,8 +112,18 @@ v3.0-0 | 2022-03-25
     - Rewrote the sections on roles and sources into a combined section that refers to account roles rather than USEF roles.
     - Deprecated the section on group notation.
 
+v2.0-7 | 2022-05-05
+"""""""""""""""""""
+
+*API v2.0 is removed.*
+
+v2.0-6 | 2022-04-26
+"""""""""""""""""""
+
+*API v2.0 is sunset.*
+
 v2.0-5 | 2022-02-13
-""""""""""""""""""""
+"""""""""""""""""""
 
 *API v2.0 is deprecated.*
 
@@ -152,6 +168,17 @@ v2.0-0 | 2020-11-14
 
 - Added REST endpoints for managing assets: `/assets/` (GET, POST) and `/asset/<id>` (GET, PATCH, DELETE).
 
+
+v1.3-14 | 2022-05-05
+""""""""""""""""""""
+
+*API v1.3 is removed.*
+
+v1.3-13 | 2022-04-26
+""""""""""""""""""""
+
+*API v1.3 is sunset.*
+
 v1.3-12 | 2022-02-13
 """"""""""""""""""""
 
@@ -243,8 +270,18 @@ v1.3-0 | 2020-01-28
 - The *postUdiEvent* endpoint now triggers scheduling jobs to be set up (rather than scheduling directly triggered by the *getDeviceMessage* endpoint)
 - The *getDeviceMessage* now queries the job queue and database for an available schedule
 
+v1.2-6 | 2022-05-05
+"""""""""""""""""""
+
+*API v1.2 is removed.*
+
+v1.2-5 | 2022-04-26
+"""""""""""""""""""
+
+*API v1.2 is sunset.*
+
 v1.2-4 | 2022-02-13
-""""""""""""""""""""
+"""""""""""""""""""
 
 *API v1.2 is deprecated.*
 
@@ -294,8 +331,18 @@ v1.2-0 | 2018-09-08
 - Added a description of the *postUdiEvent* endpoint in the Prosumer and Simulation sections
 - Added a description of the *getDeviceMessage* endpoint in the Prosumer and Simulation sections
 
+v1.1-8 | 2022-05-05
+"""""""""""""""""""
+
+*API v1.1 is removed.*
+
+v1.1-7 | 2022-04-26
+"""""""""""""""""""
+
+*API v1.1 is sunset.*
+
 v1.1-6 | 2022-02-13
-""""""""""""""""""""
+"""""""""""""""""""
 
 *API v1.1 is deprecated.*
 
@@ -352,8 +399,18 @@ v1.1-0 | 2018-07-15
 
 - Added a description of the *getPrognosis* endpoint in the Supplier section
 
+v1.0-4 | 2022-05-05
+"""""""""""""""""""
+
+*API v1.0 is removed.*
+
+v1.0-3 | 2022-04-26
+"""""""""""""""""""
+
+*API v1.0 is sunset.*
+
 v1.0-2 | 2022-02-13
-""""""""""""""""""""
+"""""""""""""""""""
 
 *API v1.0 is deprecated.*
 

diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst
@@ -100,13 +100,20 @@ which gives a response like this if the credentials are correct:
         "user_id": "<ID of the user>"
     }
 
-.. note:: Each access token has a limited lifetime, see :ref:`auth`.
+.. note:: Each access token has a limited lifetime, see :ref:`api_auth`.
 
 .. _api_deprecation:
 
 Deprecation and sunset
 ----------------------
 
+When an API feature becomes obsolete, we deprecate it.
+Deprecation of major features doesn't happen a lot, but when it does, it happens in multiple stages, during which we support clients and hosts in adapting.
+For more information on our multi-stage deprecation approach and available options for FlexMeasures hosts, see :ref:`Deprecation and sunset for hosts<api_deprecation_hosts>`.
+
+Clients
+^^^^^^^
+
 Professional API users should monitor API responses for the ``"Deprecation"`` and ``"Sunset"`` response headers [see `draft-ietf-httpapi-deprecation-header-02 <https://datatracker.ietf.org/doc/draft-ietf-httpapi-deprecation-header/>`_ and `RFC 8594 <https://www.rfc-editor.org/rfc/rfc8594>`_, respectively], so system administrators can be warned when using API endpoints that are flagged for deprecation and/or are likely to become unresponsive in the future.
 
 The deprecation header field shows an `IMF-fixdate <https://www.rfc-editor.org/rfc/rfc7231#section-7.1.1.1>`_ indicating when the API endpoint was deprecated.
@@ -139,3 +146,71 @@ Here is a client-side code example in Python (this merely prints out the depreca
                 print(f"Your request to {url} returned a sunset warning. Sunset: {content}")
             elif header == "Link" and ('rel="deprecation";' in content or 'rel="sunset";' in content):
                 print(f"Further info is available: {content}")
+
+.. _api_deprecation_hosts:
+
+Hosts
+^^^^^
+
+FlexMeasures versions go through the following stages for deprecating major features (such as API versions):
+
+- :ref:`api_deprecation_stage_1`: status 200 (OK) with relevant headers, plus a toggle to 410 (Gone) for blackout tests
+- :ref:`api_deprecation_stage_2`: status 410 (Gone), plus a toggle to 200 (OK) for sunset rollbacks
+- :ref:`api_deprecation_stage_3`: status 404 (Not Found), plus a toggle to 410 (Gone) for removal rollbacks
+- :ref:`api_deprecation_stage_4`: status 404 (Not Found), and removal of relevant endpoints
+
+Let's go over these stages in more detail.
+
+.. _api_deprecation_stage_1:
+
+Stage 1: Deprecation
+""""""""""""""""""""
+
+When upgrading to a FlexMeasures version that deprecates an API version (e.g. ``flexmeasures==0.12`` deprecates API version 2), clients will receive ``"Deprecation"`` and ``"Sunset"`` response headers [see `draft-ietf-httpapi-deprecation-header-02 <https://datatracker.ietf.org/doc/draft-ietf-httpapi-deprecation-header/>`_ and `RFC 8594 <https://www.rfc-editor.org/rfc/rfc8594>`_, respectively].
+
+Hosts should not expect every client to monitor response headers and proactively upgrade to newer API versions.
+Please make sure that your users have upgraded before you upgrade to a FlexMeasures version that sunsets an API version.
+You can do this by checking your server logs for warnings about users who are still calling deprecated endpoints.
+
+In addition, we recommend running blackout tests during the deprecation notice phase.
+You (and your users) can learn which systems need attention and how to deal with them.
+Be sure to announce these beforehand.
+Here is an example of how to run a blackout test:
+If a sunset happens in version ``0.13``, and you are hosting a version which includes the deprecation notice (e.g. ``0.12``), FlexMeasures will simulate the sunset if you set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = True`` (see :ref:`Sunset Configuration<sunset-config>`).
+During such a blackout test, clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints.
+
+.. admonition:: What is a blackout test
+   :class: info-icon
+
+   A blackout test is a planned, timeboxed event when a host will turn off a certain API or some of the API capabilities.
+   The test is meant to help developers understand the impact the retirement will have on the applications and users.
+   `Source: Platform of Trust <https://design.oftrust.net/api-migration-policies/blackout-testing>`_
+
+.. _api_deprecation_stage_2:
+
+Stage 2: Preliminary sunset
+"""""""""""""""""""""""""""
+
+When upgrading to a FlexMeasures version that sunsets an API version (e.g. ``flexmeasures==0.13`` sunsets API version 2), clients will receive ``HTTP status 410 (Gone)`` responses when calling corresponding endpoints.
+
+In case you have users that haven't upgraded yet, and would still like to upgrade FlexMeasures (to the version that officially sunsets the API version), you can.
+For a little while after sunset (usually one more minor version), we will continue to support a "sunset rollback".
+To enable this, just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = False`` and consider announcing some more blackout tests to your users, during which you can set this setting to ``True`` to reactivate the sunset.
+
+.. _api_deprecation_stage_3:
+
+Stage 3: Definitive sunset
+""""""""""""""""""""""""""
+
+After upgrading to one of the next FlexMeasures versions (e.g. ``flexmeasures==0.14``), clients that call sunset endpoints will receive ``HTTP status 404 (Not Found)`` responses.
+In case you need clients to receive the slightly more informative ``HTTP status 410 (Gone)``  for a little while longer, we will continue to support a "removal rollback".
+To enable this, just set the config setting ``FLEXMEASURES_API_SUNSET_ACTIVE = True``.
+This, just like in deprecation stages 1 and 2, leads to status 410 (Gone) responses.
+Note that ``FLEXMEASURES_API_SUNSET_ACTIVE = False`` now leads to status 404 (Not Found) responses, unlike in deprecation stages 1 and 2, where this would have lead to status 200 (OK) responses.
+
+.. _api_deprecation_stage_4:
+
+Stage 4: Removal
+""""""""""""""""
+
+After upgrading to one of the next FlexMeasures versions (e.g. ``flexmeasures==0.15``), clients that call sunset endpoints will receive ``HTTP status 404 (Not Found)`` responses.
diff --git a/documentation/api/v1.rst b/documentation/api/v1.rst
@@ -3,20 +3,5 @@
 Version 1.0
 ===========
 
-.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`.
 
-Summary
--------
-
-.. qrefflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1
-    :order: path
-    :include-empty-docstring:
-
-API Details
------------
-
-.. autoflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1
-    :order: path
-    :include-empty-docstring:
+.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.0 will stay available `here on readthedocs.org <https://flexmeasures.readthedocs.io/en/v0.12.3/api/v1.html>`_.
diff --git a/documentation/api/v1_1.rst b/documentation/api/v1_1.rst
@@ -3,20 +3,5 @@
 Version 1.1
 ===========
 
-.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`.
 
-Summary
--------
-
-.. qrefflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_1
-    :order: path
-    :include-empty-docstring:
-
-API Details
------------
-
-.. autoflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_1
-    :order: path
-    :include-empty-docstring:
+.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.1 will stay available `here on readthedocs.org <https://flexmeasures.readthedocs.io/en/v0.12.3/api/v1_1.html>`_.
diff --git a/documentation/api/v1_2.rst b/documentation/api/v1_2.rst
@@ -3,20 +3,5 @@
 Version 1.2
 ===========
 
-.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`.
 
-Summary
--------
-
-.. qrefflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_2
-    :order: path
-    :include-empty-docstring:
-
-API Details
------------
-
-.. autoflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_2
-    :order: path
-    :include-empty-docstring:
+.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.2 will stay available `here on readthedocs.org <https://flexmeasures.readthedocs.io/en/v0.12.3/api/v1_2.html>`_.
diff --git a/documentation/api/v1_3.rst b/documentation/api/v1_3.rst
@@ -3,20 +3,5 @@
 Version 1.3
 ===========
 
-.. warning:: This API version is deprecated since December 14, 2022, and will likely be sunset in February 2023. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`.
 
-Summary
--------
-
-.. qrefflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_3
-    :order: path
-    :include-empty-docstring:
-
-API Details
------------
-
-.. autoflask:: flexmeasures.app:create(env="documentation")
-    :blueprints: flexmeasures_api, flexmeasures_api_play, flexmeasures_api_v1_3
-    :order: path
-    :include-empty-docstring:
+.. note:: This API version has been sunset. Please update to :ref:`v3_0`. For more information about how FlexMeasures handles deprecation and sunsetting, see :ref:`api_deprecation`. In case your FlexMeasures host still runs ``flexmeasures<0.13``, a snapshot of documentation for API version 1.3 will stay available `here on readthedocs.org <https://flexmeasures.readthedocs.io/en/v0.12.3/api/v1_3.html>`_.