Merge branch 'main' into docker-update

.github/workflows/build.yml

@@ -0,0 +1,61 @@
+---
+name: build-docker-image
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: Build Docker Image
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        env:
+          POSTGRES_DB: flexmeasures_test
+          POSTGRES_PASSWORD: flexmeasures_test
+          POSTGRES_USER: flexmeasures_test
+        image: postgres:latest
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build Docker Image
+        run: docker build -t flexmeasures:latest -f Dockerfile .
+      - name: Generate random secret key
+        run: echo "SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_hex(24))')"
+          >> .env
+      - name: Export SQLALCHEMY_DATABASE_URI
+        run: echo "SQLALCHEMY_DATABASE_URI=postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1:5432/flexmeasures_test"
+          >> .env
+      - name: Keep running flexmeasures container in background
+        run: docker run -t -d --env-file .env --network=host --name fm-container flexmeasures:latest
+      - name: Execute database upgrade
+        run: docker exec --env-file .env fm-container flexmeasures
+          db upgrade
+      - name: Add toy user
+        run: docker exec --env-file .env fm-container flexmeasures
+          add toy-account
+      - name: Generate prices dummy data
+        run: .github/workflows/generate-dummy-price.sh
+      - name: Copy prices dummy data
+        run: docker cp prices-tomorrow.csv fm-container:/app/prices-tomorrow.csv
+      - name: Add beliefs
+        run: docker exec --env-file .env fm-container flexmeasures
+          add beliefs --sensor-id 1 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam
+      - name: Export TOMORROW
+        run: echo "TOMORROW=$(date --date="next day" '+%Y-%m-%d')"
+          >> $GITHUB_ENV
+      - name: Add schedule
+        run: docker exec --env-file .env fm-container flexmeasures
+          add schedule for-storage --sensor-id 2 --consumption-price-sensor 1 
+          --start ${TOMORROW}T07:00+01:00 --duration PT12H 
+          --soc-at-start 50% --roundtrip-efficiency 90%

.github/workflows/generate-dummy-price.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+set -x
+
+TOMORROW=$(date --date="next day" '+%Y-%m-%d')
+
+echo "Hour,Price
+${TOMORROW}T00:00:00,10
+${TOMORROW}T01:00:00,11
+${TOMORROW}T02:00:00,12
+${TOMORROW}T03:00:00,15
+${TOMORROW}T04:00:00,18
+${TOMORROW}T05:00:00,17
+${TOMORROW}T06:00:00,10.5
+${TOMORROW}T07:00:00,9
+${TOMORROW}T08:00:00,9.5
+${TOMORROW}T09:00:00,9
+${TOMORROW}T10:00:00,8.5
+${TOMORROW}T11:00:00,10
+${TOMORROW}T12:00:00,8
+${TOMORROW}T13:00:00,5
+${TOMORROW}T14:00:00,4
+${TOMORROW}T15:00:00,4
+${TOMORROW}T16:00:00,5.5
+${TOMORROW}T17:00:00,8
+${TOMORROW}T18:00:00,12
+${TOMORROW}T19:00:00,13
+${TOMORROW}T20:00:00,14
+${TOMORROW}T21:00:00,12.5
+${TOMORROW}T22:00:00,10
+${TOMORROW}T23:00:00,7" > prices-tomorrow.csv

.github/workflows/lint-and-test.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        py_version: [ '3.9' ]
+        py_version: [ "3.8", "3.9", "3.10", "3.11" ]
         include:
           - python-version: "3.9"
             coverage: yes

Makefile

@@ -70,7 +70,7 @@ install-flexmeasures:
 	pip install -e .
 
 install-pip-tools:
-	pip3 install -q "pip-tools>=7.0"
+	pip3 install -q "pip-tools>=7.2"
 
 install-docs-dependencies:
 	pip install -r requirements/${PYV}/docs.txt
@@ -79,27 +79,17 @@ freeze-deps:
 	make ensure-deps-folder
 	make install-pip-tools
 	pip-compile -o requirements/${PYV}/app.txt requirements/app.in
-	# Create app.txt to create constraints for test.txt and dev.txt
-	cat requirements/${PYV}/app.txt > requirements/app.txt
-	pip-compile -o requirements/${PYV}/test.txt requirements/test.in
-	cat requirements/${PYV}/test.txt > requirements/test.txt
-	pip-compile -o requirements/${PYV}/dev.txt requirements/dev.in
-	pip-compile -o requirements/${PYV}/docs.txt requirements/docs.in
-	rm requirements/app.txt
-	rm requirements/test.txt
+	pip-compile -c requirements/${PYV}/app.txt -o requirements/${PYV}/test.txt requirements/test.in
+	pip-compile -c requirements/${PYV}/app.txt -c requirements/${PYV}/test.txt -o requirements/${PYV}/dev.txt requirements/dev.in
+	pip-compile -c requirements/${PYV}/app.txt -o requirements/${PYV}/docs.txt requirements/docs.in
 
 upgrade-deps:
 	make ensure-deps-folder
 	make install-pip-tools
 	pip-compile --upgrade -o requirements/${PYV}/app.txt requirements/app.in
-	# Create app.txt to create constraints for test.txt and dev.txt
-	cat requirements/${PYV}/app.txt > requirements/app.txt
-	pip-compile --upgrade -o requirements/${PYV}/test.txt requirements/test.in
-	cat requirements/${PYV}/test.txt > requirements/test.txt
-	pip-compile --upgrade -o requirements/${PYV}/dev.txt requirements/dev.in
-	pip-compile --upgrade -o requirements/${PYV}/docs.txt requirements/docs.in
-	rm requirements/app.txt
-	rm requirements/test.txt
+	pip-compile --upgrade -c requirements/${PYV}/app.txt -o requirements/${PYV}/test.txt requirements/test.in
+	pip-compile --upgrade -c requirements/${PYV}/app.txt -c requirements/${PYV}/test.txt -o requirements/${PYV}/dev.txt requirements/dev.in
+	pip-compile --upgrade -c requirements/${PYV}/app.txt -o requirements/${PYV}/docs.txt requirements/docs.in
 
 ifneq ($(skip-test), yes)
 	make test

documentation/changelog.rst

@@ -9,14 +9,16 @@ v0.16.0 | September XX, 2023
 New features
 -------------
 
-* Introduce new reporter to compute profit/loss due to electricity flows: `ProfitOrLossReporter` [see `PR #808 <https://github.com/FlexMeasures/flexmeasures/pull/808>`_]
+* Introduce new reporter to compute profit/loss due to electricity flows: `ProfitOrLossReporter` [see `PR #808 <https://github.com/FlexMeasures/flexmeasures/pull/808>`_ and `PR #844 <https://github.com/FlexMeasures/flexmeasures/pull/844>`_]
 * Charts visible in the UI can be exported to PNG or SVG formats in a more automated fashion, using the new CLI command flexmeasures show chart [see `PR #833 <https://github.com/FlexMeasures/flexmeasures/pull/833>`_]
 
 Infrastructure / Support
 ----------------------
 
 * Allow additional datetime conversions to quantitative time units, specifically, from timezone-naive and/or dayfirst datetimes, which can be useful when importing data [see `PR #831 <https://github.com/FlexMeasures/flexmeasures/pull/831>`_]
 * Add a new tutorial to explain the use of the `AggregatorReporter` to compute the headroom and the `ProfitOrLossReporter` to compute the cost of running a process [see `PR #825 <https://github.com/FlexMeasures/flexmeasures/pull/825>`_]
+* Have our CI pipeline (GitHub Actions) build the Docker image and make a schedule [see `PR #800 <https://github.com/FlexMeasures/flexmeasures/pull/800>`_]
+* Test all supported Python versions in our CI pipeline (GitHub Actions) [see `PR #847 <https://github.com/FlexMeasures/flexmeasures/pull/847>`_]
 
 v0.15.1 | August 28, 2023
 ============================

flexmeasures/data/models/reporting/profit.py

@@ -72,7 +72,8 @@ def _compute_report(
         loss_is_positive: bool = self._config.get("loss_is_positive", False)
 
         input_sensor: Sensor = input[0]["sensor"]  # power or energy sensor
-        input_source: Sensor = input[0].get("source", None)
+        input_source: Sensor = input[0].get("sources", None)
+
         timezone = input_sensor.timezone
 
         if belief_time is None:
@@ -85,6 +86,8 @@ def _compute_report(
                 event_ends_before=end,
                 beliefs_before=belief_time,
                 resolution=input_sensor.event_resolution,
+                most_recent_beliefs_only=True,
+                one_deterministic_belief_per_event=True,
             )
         )
         production_price = production_price.tz_convert(timezone)
@@ -94,6 +97,8 @@ def _compute_report(
                 event_ends_before=end,
                 beliefs_before=belief_time,
                 resolution=input_sensor.event_resolution,
+                most_recent_beliefs_only=True,
+                one_deterministic_belief_per_event=True,
             )
         )
         consumption_price = consumption_price.tz_convert(timezone)
@@ -105,6 +110,8 @@ def _compute_report(
                 event_ends_before=end,
                 beliefs_before=belief_time,
                 source=input_source,
+                most_recent_beliefs_only=True,
+                one_deterministic_belief_per_event=True,
             )
         )
 

flexmeasures/data/services/scheduling.py

@@ -103,7 +103,6 @@ def make_schedule(
     flex_context: dict | None = None,
     flex_config_has_been_deserialized: bool = False,
 ) -> bool:
-
     """
     This function computes a schedule. It returns True if it ran successfully.
 
@@ -155,8 +154,8 @@ def make_schedule(
     )
 
     # saving info on the job, so the API for a job can look the data up
-    data_source_info["id"] = data_source.id
     if rq_job:
+        data_source_info["id"] = data_source.id
         rq_job.meta["data_source_info"] = data_source_info
         rq_job.save_meta()
 
@@ -278,26 +277,21 @@ def handle_scheduling_exception(job, exc_type, exc_value, traceback):
     job.save_meta()
 
 
-def get_data_source_for_job(job: Job | None) -> DataSource | None:
+def get_data_source_for_job(job: Job) -> DataSource | None:
     """
     Try to find the data source linked by this scheduling job.
 
-    We expect that enough info on the source was placed in the meta dict.
-    This only happened with v0.12. For a transition period, we might have to support older jobs who haven't got that info.
-    TODO: We should expect a job, once API v1.3 is deprecated.
+    We expect that enough info on the source was placed in the meta dict, either:
+    - the DataSource ID itself (i.e. the normal situation), or
+    - enough info to facilitate a DataSource query (as a fallback).
     """
-    data_source_info = None
-    if job:
-        data_source_info = job.meta.get("data_source_info")
-        if (
-            data_source_info and "id" in data_source_info
-        ):  # this is the expected outcome
-            return DataSource.query.get(data_source_info["id"])
+    data_source_info = job.meta.get("data_source_info")
+    if data_source_info and "id" in data_source_info:
+        # this is the expected outcome
+        return DataSource.query.get(data_source_info["id"])
     if data_source_info is None:
-        data_source_info = dict(name="Seita", model="StorageScheduler")
-        # TODO: change to raise later (v0.13) - all scheduling jobs now get full info
-        current_app.logger.warning(
-            "Looking up scheduling data without knowing full data_source_info (version). This is deprecated soon. Please specify a job id as event or switch to API v3."
+        raise ValueError(
+            "Cannot look up scheduling data without knowing the full data_source_info (version)."
         )
     scheduler_sources = (
         DataSource.query.filter_by(

requirements/3.10/dev.txt

@@ -2,22 +2,22 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --output-file=requirements/3.10/dev.txt requirements/dev.in
+#    pip-compile --constraint=requirements/3.10/app.txt --constraint=requirements/3.10/test.txt --output-file=requirements/3.10/dev.txt requirements/dev.in
 #
 black==22.3.0
     # via -r requirements/dev.in
 cfgv==3.4.0
     # via pre-commit
 click==8.1.7
     # via
-    #   -c requirements/app.txt
-    #   -c requirements/test.txt
+    #   -c requirements/3.10/app.txt
+    #   -c requirements/3.10/test.txt
     #   black
 distlib==0.3.7
     # via virtualenv
 filelock==3.12.2
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   virtualenv
 flake8==4.0.1
     # via -r requirements/dev.in
@@ -37,8 +37,8 @@ nodeenv==1.8.0
     # via pre-commit
 packaging==23.1
     # via
-    #   -c requirements/app.txt
-    #   -c requirements/test.txt
+    #   -c requirements/3.10/app.txt
+    #   -c requirements/3.10/test.txt
     #   setuptools-scm
 pathspec==0.11.2
     # via black
@@ -58,19 +58,19 @@ pytest-runner==6.0.0
     # via -r requirements/dev.in
 pyyaml==6.0.1
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   pre-commit
 setuptools-scm==7.1.0
     # via -r requirements/dev.in
 tomli==2.0.1
     # via
-    #   -c requirements/test.txt
+    #   -c requirements/3.10/test.txt
     #   black
     #   mypy
     #   setuptools-scm
 typing-extensions==4.7.1
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   mypy
     #   setuptools-scm
 virtualenv==20.24.3

requirements/3.10/docs.txt

@@ -2,21 +2,21 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --output-file=requirements/3.10/docs.txt requirements/docs.in
+#    pip-compile --constraint=requirements/3.10/app.txt --output-file=requirements/3.10/docs.txt requirements/docs.in
 #
 alabaster==0.7.13
     # via sphinx
 babel==2.12.1
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   sphinx
 certifi==2023.7.22
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   requests
 charset-normalizer==3.2.0
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   requests
 docutils==0.18.1
     # via
@@ -25,33 +25,33 @@ docutils==0.18.1
     #   sphinx-tabs
 idna==3.4
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   requests
 imagesize==1.4.1
     # via sphinx
 jinja2==3.1.2
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   sphinx
 markupsafe==2.1.3
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   jinja2
 packaging==23.1
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   sphinx
 pygments==2.16.1
     # via
     #   sphinx
     #   sphinx-tabs
 requests==2.31.0
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   sphinx
 six==1.16.0
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   sphinxcontrib-httpdomain
 snowballstemmer==2.2.0
     # via sphinx
@@ -95,5 +95,5 @@ sphinxcontrib-serializinghtml==1.1.9
     # via sphinx
 urllib3==2.0.4
     # via
-    #   -c requirements/app.txt
+    #   -c requirements/3.10/app.txt
     #   requests