Merge branch 'main' into benchmark-util

Signed-off-by: Nicolas Höning <nicolas@seita.nl>

.github/workflows/build.yml

@@ -0,0 +1,61 @@
+---
+name: build-docker-image
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: Build Docker Image
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        env:
+          POSTGRES_DB: flexmeasures_test
+          POSTGRES_PASSWORD: flexmeasures_test
+          POSTGRES_USER: flexmeasures_test
+        image: postgres:latest
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build Docker Image
+        run: docker build -t flexmeasures:latest -f Dockerfile .
+      - name: Generate random secret key
+        run: echo "SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_hex(24))')"
+          >> .env
+      - name: Export SQLALCHEMY_DATABASE_URI
+        run: echo "SQLALCHEMY_DATABASE_URI=postgresql://flexmeasures_test:flexmeasures_test@127.0.0.1:5432/flexmeasures_test"
+          >> .env
+      - name: Keep running flexmeasures container in background
+        run: docker run -t -d --env-file .env --network=host --name fm-container flexmeasures:latest
+      - name: Execute database upgrade
+        run: docker exec --env-file .env fm-container flexmeasures
+          db upgrade
+      - name: Add toy user
+        run: docker exec --env-file .env fm-container flexmeasures
+          add toy-account
+      - name: Generate prices dummy data
+        run: .github/workflows/generate-dummy-price.sh
+      - name: Copy prices dummy data
+        run: docker cp prices-tomorrow.csv fm-container:/app/prices-tomorrow.csv
+      - name: Add beliefs
+        run: docker exec --env-file .env fm-container flexmeasures
+          add beliefs --sensor-id 1 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam
+      - name: Export TOMORROW
+        run: echo "TOMORROW=$(date --date="next day" '+%Y-%m-%d')"
+          >> $GITHUB_ENV
+      - name: Add schedule
+        run: docker exec --env-file .env fm-container flexmeasures
+          add schedule for-storage --sensor-id 2 --consumption-price-sensor 1 
+          --start ${TOMORROW}T07:00+01:00 --duration PT12H 
+          --soc-at-start 50% --roundtrip-efficiency 90%

.github/workflows/generate-dummy-price.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+set -x
+
+TOMORROW=$(date --date="next day" '+%Y-%m-%d')
+
+echo "Hour,Price
+${TOMORROW}T00:00:00,10
+${TOMORROW}T01:00:00,11
+${TOMORROW}T02:00:00,12
+${TOMORROW}T03:00:00,15
+${TOMORROW}T04:00:00,18
+${TOMORROW}T05:00:00,17
+${TOMORROW}T06:00:00,10.5
+${TOMORROW}T07:00:00,9
+${TOMORROW}T08:00:00,9.5
+${TOMORROW}T09:00:00,9
+${TOMORROW}T10:00:00,8.5
+${TOMORROW}T11:00:00,10
+${TOMORROW}T12:00:00,8
+${TOMORROW}T13:00:00,5
+${TOMORROW}T14:00:00,4
+${TOMORROW}T15:00:00,4
+${TOMORROW}T16:00:00,5.5
+${TOMORROW}T17:00:00,8
+${TOMORROW}T18:00:00,12
+${TOMORROW}T19:00:00,13
+${TOMORROW}T20:00:00,14
+${TOMORROW}T21:00:00,12.5
+${TOMORROW}T22:00:00,10
+${TOMORROW}T23:00:00,7" > prices-tomorrow.csv

.github/workflows/lint-and-test.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        py_version: [ '3.9' ]
+        py_version: [ "3.8", "3.9", "3.10", "3.11" ]
         include:
           - python-version: "3.9"
             coverage: yes
@@ -38,13 +38,13 @@ jobs:
         run: |
           git fetch --prune --unshallow
           git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-      - name: "Caching dependencies (txt)"
+      - name: "Caching for dependencies (.txt) - restore existing or ensure new cache will be made"
         uses: actions/cache@v2
         id: cache
         with:
           path: ${{ env.pythonLocation }}
           # manually disable a cache if needed by (re)setting CACHE_DATE
-          key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ SECRETS.CACHE_DATE }}-${{ hashFiles('**/requirements/**.txt') }}
+          key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ SECRETS.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - run: |

.readthedocs.yaml

@@ -21,8 +21,8 @@ build:
 
 python: 
   install:
-    - requirements: requirements/app.txt
-    - requirements: requirements/docs.txt
+    - requirements: requirements/3.9/app.txt
+    - requirements: requirements/3.9/docs.txt
 
 
 sphinx:

.vscode/spellright.dict

@@ -266,3 +266,4 @@ idx
 subprocess
 usr
 ±
+UNPROCESSABLE_ENTITY

Dockerfile

@@ -1,5 +1,5 @@
-FROM ubuntu:focal
- 
+FROM ubuntu:22.04
+
 ENV DEBIAN_FRONTEND noninteractive
 ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8
@@ -12,7 +12,10 @@ WORKDIR /app
 COPY requirements /app/requirements
 
 # py dev tooling
-RUN python3 -m pip install --no-cache-dir  --upgrade pip && python3 --version && pip3 install --no-cache-dir --upgrade setuptools && pip3 install highspy && pip3 install --no-cache-dir -r requirements/app.txt -r requirements/dev.txt -r requirements/test.txt
+RUN python3 -m pip install --no-cache-dir --upgrade pip && python3 --version && \
+    pip3 install --no-cache-dir --upgrade setuptools && pip3 install highspy && \
+    PYV=$(python3 -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") && \
+    pip3 install --no-cache-dir -r requirements/$PYV/app.txt -r requirements/$PYV/dev.txt -r requirements/$PYV/test.txt
 
 # Copy code and meta/config data
 COPY setup.* .flaskenv wsgi.py /app/
@@ -34,4 +37,4 @@ CMD [ \
     # another request is taking a long time to complete.
     "--workers", "2", "--threads", "4", \
     "wsgi:application" \
-]
+    ]

Makefile

@@ -1,6 +1,10 @@
+# Check Python major and minor version
+# For more information, see https://stackoverflow.com/a/22105036
+PYV = $(shell python -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)")
+
 # Note: use tabs
 # actions which are virtual, i.e. not a script
-.PHONY: install install-for-dev install-for-test install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db
+.PHONY: install install-for-dev install-for-test install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db cli-autocomplete
 
 
 # ---- Development ---
@@ -35,14 +39,15 @@ install: install-deps install-flexmeasures
 
 install-for-dev:
 	make freeze-deps
-	pip-sync requirements/app.txt requirements/dev.txt requirements/test.txt
+	make ensure-deps-folder
+	pip-sync requirements/${PYV}/app.txt requirements/${PYV}/dev.txt requirements/${PYV}/test.txt
 	make install-flexmeasures
 
 install-for-test:
 	make install-pip-tools
 # Pass pinned=no if you want to test against latest stable packages, default is our pinned dependency set
 ifneq ($(pinned), no)
-	pip-sync requirements/app.txt requirements/test.txt
+	pip-sync requirements/${PYV}/app.txt requirements/${PYV}/test.txt
 else
 	# cutting off the -c inter-layer dependency (that's pip-tools specific)
 	tail -n +3 requirements/test.in >> temp-test.in
@@ -56,7 +61,7 @@ install-deps:
 	make freeze-deps
 # Pass pinned=no if you want to test against latest stable packages, default is our pinned dependency set
 ifneq ($(pinned), no)
-	pip-sync requirements/app.txt
+	pip-sync requirements/${PYV}/app.txt
 else
 	pip install --upgrade -r requirements/app.in
 endif
@@ -65,26 +70,30 @@ install-flexmeasures:
 	pip install -e .
 
 install-pip-tools:
-	pip3 install -q "pip-tools>=7.0"
+	pip3 install -q "pip-tools>=7.2"
 
 install-docs-dependencies:
-	pip install -r requirements/docs.txt
+	pip install -r requirements/${PYV}/docs.txt
 
 freeze-deps:
+	make ensure-deps-folder
 	make install-pip-tools
-	pip-compile -o requirements/app.txt requirements/app.in
-	pip-compile -o requirements/test.txt requirements/test.in
-	pip-compile -o requirements/dev.txt requirements/dev.in
-	pip-compile -o requirements/docs.txt requirements/docs.in
+	pip-compile -o requirements/${PYV}/app.txt requirements/app.in
+	pip-compile -c requirements/${PYV}/app.txt -o requirements/${PYV}/test.txt requirements/test.in
+	pip-compile -c requirements/${PYV}/app.txt -c requirements/${PYV}/test.txt -o requirements/${PYV}/dev.txt requirements/dev.in
+	pip-compile -c requirements/${PYV}/app.txt -o requirements/${PYV}/docs.txt requirements/docs.in
 
 upgrade-deps:
+	make ensure-deps-folder
 	make install-pip-tools
-	pip-compile --upgrade -o requirements/app.txt requirements/app.in
-	pip-compile --upgrade -o requirements/test.txt requirements/test.in
-	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
-	pip-compile --upgrade -o requirements/docs.txt requirements/docs.in
-	make test
+	pip-compile --upgrade -o requirements/${PYV}/app.txt requirements/app.in
+	pip-compile --upgrade -c requirements/${PYV}/app.txt -o requirements/${PYV}/test.txt requirements/test.in
+	pip-compile --upgrade -c requirements/${PYV}/app.txt -c requirements/${PYV}/test.txt -o requirements/${PYV}/dev.txt requirements/dev.in
+	pip-compile --upgrade -c requirements/${PYV}/app.txt -o requirements/${PYV}/docs.txt requirements/docs.in
 
+ifneq ($(skip-test), yes)
+	make test
+endif
 
 # ---- Data ----
 
@@ -104,5 +113,11 @@ show-data-model:
 	# Use --help to learn more. 
 	./flexmeasures/data/scripts/visualize_data_model.py --uml
 
+ensure-deps-folder:
+	mkdir -p requirements/${PYV}
+
 clean-db:
 	./flexmeasures/data/scripts/clean_database.sh ${db_name} ${db_user}
+
+cli-autocomplete:
+	./flexmeasures/cli/scripts/add_scripts_path.sh ${extension}

ci/Dockerfile.update

@@ -0,0 +1,17 @@
+ARG PYTHON_VERSION
+
+FROM python:${PYTHON_VERSION}-slim-bookworm as update
+
+# Install dependencies
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy the source code
+
+COPY . /app
+WORKDIR /app
+
+CMD ["python", "--version"]

ci/run_mypy.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 pip install --upgrade 'mypy>=0.902'
-pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate
+pip install types-pytz types-requests types-Flask types-click types-redis types-tzlocal types-python-dateutil types-setuptools types-tabulate types-PyYAML
 # We are checking python files which have type hints, and leave out bigger issues we made issues for
 # * data/scripts: We'll remove legacy code: https://trello.com/c/1wEnHOkK/7-remove-custom-data-scripts
 # * data/models and data/services: https://trello.com/c/rGxZ9h2H/540-makequery-call-signature-is-incoherent

ci/update-packages.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+######################################################################
+# This script sets up docker environments for supported Python versions
+# for updating packages in each of them.
+#
+# To upgrade, add "upgrade" as parameter.
+#
+# To execute this script, cd into the `ci` directory, then call from there.
+######################################################################
+
+set -e
+set -x
+
+PYTHON_VERSIONS=(3.8 3.9 3.10 3.11)
+
+# check if we will upgrade or just freeze
+UPDATE_CMD=freeze-deps
+if [ "$1" == "upgrade" ]; then
+  UPDATE_CMD=upgrade-deps
+  echo "Going to upgrade dependencies with make $UPDATE_CMD ..."
+else
+  echo "Going to freeze dependencies with make $UPDATE_CMD..."
+fi
+
+# Check if docker is installed
+if ! [ -x "$(command -v docker)" ]; then
+  echo "Docker is not installed. Please install docker and try again."
+  exit 1
+fi
+
+# Check if we can run docker without sudo
+if ! docker ps > /dev/null 2>&1; then
+  echo "Docker is not running without sudo. Please add your user to the docker group and try again."
+  echo "You may use the following command to do so:"
+  echo "sudo usermod -aG docker $USER"
+  echo "You will need to log out and log back in for this to take effect."
+  exit 1
+fi
+
+SOURCE_DIR=$(pwd)/../
+
+TEMP_DIR=$(mktemp -d)
+
+# Copy the build files to the temp directory
+cp -r ../ci $TEMP_DIR/ci
+cp -r ../requirements $TEMP_DIR/requirements
+cp -r ../Makefile $TEMP_DIR
+
+cd $TEMP_DIR
+
+
+for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"
+do
+    # Check if container exists and remove it
+    docker container inspect flexmeasures-update-packages-$PYTHON_VERSION > /dev/null 2>&1 && docker rm --force flexmeasures-update-packages-$PYTHON_VERSION
+    # Build the docker image
+    docker build --build-arg=PYTHON_VERSION=$PYTHON_VERSION -t flexmeasures-update-packages:$PYTHON_VERSION . -f ci/Dockerfile.update
+    # Build flexmeasures
+    # We are disabling running tests here, because we only want to update the packages. Running tests would require us to setup a postgres database,
+    # which is not necessary for updating packages.
+    docker run --name flexmeasures-update-packages-$PYTHON_VERSION -it flexmeasures-update-packages:$PYTHON_VERSION make $UPDATE_CMD skip-test=yes
+    # Copy the requirements to the source directory
+    docker cp flexmeasures-update-packages-$PYTHON_VERSION:/app/requirements/$PYTHON_VERSION $SOURCE_DIR/requirements/
+    # Remove the container
+    docker rm flexmeasures-update-packages-$PYTHON_VERSION
+    # Remove the image
+    docker rmi flexmeasures-update-packages:$PYTHON_VERSION
+done
+
+# Clean up docker builder cache
+echo "You can clean up the docker builder cache with the following command:"
+echo "docker builder prune --all -f"
+
+# Remove the temp directory
+rm -rf $TEMP_DIR
+
+# Return to the ci directory (in case you want to rerun this script)
+cd $SOURCE_DIR
+cd ci

documentation/api/change_log.rst

@@ -3,7 +3,12 @@
 API change log
 ===============
 
-.. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL, allowing developers to upgrade at their own pace.
+.. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace.
+
+v3.0-12 | 2023-09-20
+""""""""""""""""""""
+
+- Introduced the ``power-capacity`` field under ``flex-model``, and the ``site-power-capacity`` field under ``flex-context``, for `/sensors/<id>/schedules/trigger` (POST).
 
 v3.0-11 | 2023-08-02
 """"""""""""""""""""

documentation/api/notation.rst

@@ -197,6 +197,7 @@ Here are the three types of flexibility models you can expect to be built-in:
    - ``roundtrip-efficiency`` (defaults to 100%)
    - ``storage-efficiency`` (defaults to 100%) [#]_
    - ``prefer-charging-sooner`` (defaults to True, also signals a preference to discharge later)
+   - ``power-capacity`` (defaults to the Sensor attribute ``capacity_in_mw``)
 
     .. [#] The storage efficiency (e.g. 95% or 0.95) to use for the schedule is applied over each time step equal to the sensor resolution. For example, a storage efficiency of 95 percent per (absolute) day, for scheduling a 1-hour resolution sensor, should be passed as a storage efficiency of :math:`0.95^{1/24} = 0.997865`.
 
@@ -236,6 +237,7 @@ With the flexibility context, we aim to describe the system in which the flexibl
 - ``inflexible-device-sensors`` ― power sensors that are relevant, but not flexible, such as a sensor recording rooftop solar power connected behind the main meter, whose production falls under the same contract as the flexible device(s) being scheduled
 - ``consumption-price-sensor`` ― the sensor which defines costs/revenues of consuming energy
 - ``production-price-sensor`` ― the sensor which defines cost/revenues of producing energy
+- ``site-power-capacity`` (defaults to the Asset attribute ``capacity_in_mw``)
 
 These should be independent on the asset type and consequently also do not depend on which scheduling algorithm is being used.