From cb5f5cef9f10fa1db80c39fcf1fa93051efbbe5e Mon Sep 17 00:00:00 2001 From: Nischay Mamidi <15060256+Nischay-Pro@users.noreply.github.com> Date: Fri, 15 Sep 2023 07:42:08 -0400 Subject: [PATCH 1/3] installation: added docker script to auto update dependencies for all Py versions (#843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added ci script to auto update images Signed-off-by: Nischay Mamidi * more compact and quicker code Signed-off-by: Nischay Mamidi * add comment for skip-tests flag * empty commit to check glitch * add documentation for devs Signed-off-by: Nicolas Höning * add changelog entry Signed-off-by: Nicolas Höning * checking requirements back to main status Signed-off-by: Nicolas Höning --------- Signed-off-by: Nischay Mamidi Signed-off-by: Nicolas Höning Co-authored-by: Nicolas Höning --- Makefile | 2 + ci/Dockerfile.update | 17 ++++++ ci/update-packages.sh | 61 +++++++++++++++++++++ documentation/changelog.rst | 3 +- documentation/dev/dependency-management.rst | 43 +++++++++++++++ documentation/index.rst | 1 + 6 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 ci/Dockerfile.update create mode 100755 ci/update-packages.sh create mode 100644 documentation/dev/dependency-management.rst diff --git a/Makefile b/Makefile index e4c9a7878..99c115bce 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,9 @@ upgrade-deps: pip-compile --upgrade -c requirements/${PYV}/app.txt -c requirements/${PYV}/test.txt -o requirements/${PYV}/dev.txt requirements/dev.in pip-compile --upgrade -c requirements/${PYV}/app.txt -o requirements/${PYV}/docs.txt requirements/docs.in +ifneq ($(skip-test), yes) make test +endif # ---- Data ---- diff --git a/ci/Dockerfile.update b/ci/Dockerfile.update new file mode 100644 index 000000000..3138ef3fe --- /dev/null +++ b/ci/Dockerfile.update @@ -0,0 +1,17 @@ +ARG PYTHON_VERSION + +FROM python:${PYTHON_VERSION}-slim-bookworm as update + +# Install dependencies + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Copy the source code + +COPY . /app +WORKDIR /app + +CMD ["python", "--version"] \ No newline at end of file diff --git a/ci/update-packages.sh b/ci/update-packages.sh new file mode 100755 index 000000000..c3c803296 --- /dev/null +++ b/ci/update-packages.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +###################################################################### +# This script sets up the docker environment for updating packages +###################################################################### + +set -e +set -x + +# Check if docker is installed +if ! [ -x "$(command -v docker)" ]; then + echo "Docker is not installed. Please install docker and try again." + exit 1 +fi + +# Check if we can run docker without sudo +if ! docker ps > /dev/null 2>&1; then + echo "Docker is not running without sudo. Please add your user to the docker group and try again." + echo "You may use the following command to do so:" + echo "sudo usermod -aG docker $USER" + echo "You will need to log out and log back in for this to take effect." + exit 1 +fi + +SOURCE_DIR=$(pwd)/../ + +TEMP_DIR=$(mktemp -d) + +# Copy the build files to the temp directory +cp -r ../ci $TEMP_DIR/ci +cp -r ../requirements $TEMP_DIR/requirements +cp -r ../Makefile $TEMP_DIR + +cd $TEMP_DIR + +PYTHON_VERSIONS=(3.8 3.9 3.10 3.11) + +for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}" +do + # Check if container exists and remove it + docker container inspect flexmeasures-update-packages-$PYTHON_VERSION > /dev/null 2>&1 && docker rm --force flexmeasures-update-packages-$PYTHON_VERSION + # Build the docker image + docker build --build-arg=PYTHON_VERSION=$PYTHON_VERSION -t flexmeasures-update-packages:$PYTHON_VERSION . -f ci/Dockerfile.update + # Build flexmeasures + # We are disabling running tests here, because we only want to update the packages. Running tests would require us to setup a postgres database, + # which is not necessary for updating packages. + docker run --name flexmeasures-update-packages-$PYTHON_VERSION -it flexmeasures-update-packages:$PYTHON_VERSION make upgrade-deps skip-test=yes + # Copy the requirements to the source directory + docker cp flexmeasures-update-packages-$PYTHON_VERSION:/app/requirements/$PYTHON_VERSION $SOURCE_DIR/requirements/ + # Remove the container + docker rm flexmeasures-update-packages-$PYTHON_VERSION + # Remove the image + docker rmi flexmeasures-update-packages:$PYTHON_VERSION +done + +# Clean up docker builder cache +echo "You can clean up the docker builder cache with the following command:" +echo "docker builder prune --all -f" + +# Remove the temp directory +rm -rf $TEMP_DIR \ No newline at end of file diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 93de22424..b2c41be2e 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,8 +17,9 @@ Infrastructure / Support * Allow additional datetime conversions to quantitative time units, specifically, from timezone-naive and/or dayfirst datetimes, which can be useful when importing data [see `PR #831 `_] * Add a new tutorial to explain the use of the `AggregatorReporter` to compute the headroom and the `ProfitOrLossReporter` to compute the cost of running a process [see `PR #825 `_] -* Have our CI pipeline (GitHub Actions) build the Docker image and make a schedule [see `PR #800 `_] +* Script to update dependencies across supported Python versions [see `PR #843 `_] * Test all supported Python versions in our CI pipeline (GitHub Actions) [see `PR #847 `_] +* Have our CI pipeline (GitHub Actions) build the Docker image and make a schedule [see `PR #800 `_] v0.15.1 | August 28, 2023 ============================ diff --git a/documentation/dev/dependency-management.rst b/documentation/dev/dependency-management.rst new file mode 100644 index 000000000..c0645469a --- /dev/null +++ b/documentation/dev/dependency-management.rst @@ -0,0 +1,43 @@ +Dependency Management +======================= + +Requirements +------------- + +FlexMeasures is built on the shoulder of giants, namely other open source libraries. +Look into the `requirements` folder to see what is required to run FlexMeasures (`app.in`) or to test it, or to build this documentation. + +The `.in` files specify our general demands, and in `.txt` files, we keep a set of pinned dependency versions, so we can all work on the same background (crucial to compare behavior of installations to each other). + +To install these pinned requirements, run: + +.. code-block:: bash + + $ make install-for-dev + +Check out `Makefile` for other useful commands, but this should get you going. + +To upgrade the pinned versions, we can run: + + +.. code-block:: bash + + $ make upgrade-deps + + +Python versions +---------------- + +In addition, we support a range of Python versions (as you can see in the `requirements` folder. + +Now ― you probably have only one Python version installed. Let's say you add a dependency, or update the minimum required version. How to update the pinned sets of requirements across all Python versions? + +.. code-block:: bash + + $ cd ci; ./update-packages.sh + +This script will use docker to do these upgrades per Python version. + +Still, we'd also like to be able to test FlexMeasures across all these versions. +We've added that capability to our CI pipeline (GitHub Actions), so you could clone it an make a PR, in order to run them. + diff --git a/documentation/index.rst b/documentation/index.rst index 102aafaee..a23ad5a8b 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -247,6 +247,7 @@ The platform operator of FlexMeasures can be an Aggregator. dev/ci dev/auth dev/docker-compose + dev/dependency-management .. autosummary:: From f3b5c4d08b056b4624db5ae45a56258b644407c0 Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Fri, 15 Sep 2023 15:05:14 +0200 Subject: [PATCH 2/3] fix(docs): IDs (#852) Signed-off-by: F.N. Claessen --- documentation/tut/building_uis.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/tut/building_uis.rst b/documentation/tut/building_uis.rst index 6a7660690..7e6fc2211 100644 --- a/documentation/tut/building_uis.rst +++ b/documentation/tut/building_uis.rst @@ -116,7 +116,7 @@ Load asset information Similarly, we can load asset information. Say we have a variable ``accountId`` and we want to show which assets FlexMeasures administrates for that account. -For the example below, we've used the Id of the account from our toy tutorial, see :ref:`toy tutorial`. +For the example below, we've used the ID of the account from our toy tutorial, see :ref:`toy tutorial`. .. code-block:: html @@ -135,7 +135,7 @@ For the example below, we've used the Id of the account from our toy tutorial, s Asset name - Id + ID Latitude Longitude @@ -278,13 +278,13 @@ Now let's call this function when the HTML page is opened, to embed our chart: var params = new URLSearchParams(); params.append("event_starts_after", '2022-01-01T00:00+01'); - embedChart(params, authToken, 3, '#sensor-chart'); + embedChart(params, authToken, 1, '#sensor-chart'); }) } } The parameters we pass in describe what we want to see: all data for sensor 3 since 2022. -If you followed our :ref:`toy tutorial` on a fresh FlexMeasures installation, sensor 3 contains market prices (authenticate with the toy-user to gain access). +If you followed our :ref:`toy tutorial` on a fresh FlexMeasures installation, sensor 1 contains market prices (authenticate with the toy-user to gain access). The result looks like this in your browser: From 8204b169a8b2ba71b81eb051cb15af77db98da67 Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Fri, 15 Sep 2023 17:37:37 +0200 Subject: [PATCH 3/3] Feat/interpolate instantaneous sensor charts (#851) * fix: sensor chart with instantaneous sensor readings Signed-off-by: F.N. Claessen * docs: changelog entry Signed-off-by: F.N. Claessen * fix(docs): correct PR number Signed-off-by: F.N. Claessen * feat: validate attribute before storing Signed-off-by: F.N. Claessen --------- Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + flexmeasures/cli/data_edit.py | 4 ++++ flexmeasures/data/models/charts/belief_charts.py | 10 ++++++++-- flexmeasures/data/schemas/attributes.py | 9 +++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 flexmeasures/data/schemas/attributes.py diff --git a/documentation/changelog.rst b/documentation/changelog.rst index b2c41be2e..f83f8bbe0 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -11,6 +11,7 @@ New features * Introduce new reporter to compute profit/loss due to electricity flows: `ProfitOrLossReporter` [see `PR #808 `_ and `PR #844 `_] * Charts visible in the UI can be exported to PNG or SVG formats in a more automated fashion, using the new CLI command flexmeasures show chart [see `PR #833 `_] +* Sensor charts showing instantaneous observations can be interpolated by setting the ``interpolate`` sensor attribute to one of the `supported Vega-Lite interpolation methods `_ [see `PR #851 `_] Infrastructure / Support ---------------------- diff --git a/flexmeasures/cli/data_edit.py b/flexmeasures/cli/data_edit.py index 63b104fb9..747e0c51f 100644 --- a/flexmeasures/cli/data_edit.py +++ b/flexmeasures/cli/data_edit.py @@ -14,6 +14,7 @@ from flexmeasures import Sensor from flexmeasures.data import db +from flexmeasures.data.schemas.attributes import validate_special_attributes from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.sensors import SensorIdField from flexmeasures.data.models.generic_assets import GenericAsset @@ -129,6 +130,9 @@ def edit_attribute( attribute_null_value=attribute_null_value, ) + # Some attributes with special in meaning in FlexMeasures must pass validation + validate_special_attributes(attribute_key, attribute_value) + # Set attribute for asset in assets: asset.attributes[attribute_key] = attribute_value diff --git a/flexmeasures/data/models/charts/belief_charts.py b/flexmeasures/data/models/charts/belief_charts.py index c93f2e415..af9946844 100644 --- a/flexmeasures/data/models/charts/belief_charts.py +++ b/flexmeasures/data/models/charts/belief_charts.py @@ -45,14 +45,20 @@ def bar_chart( event_ends_before.timestamp() * 10**3, ] } + mark_type = "bar" + mark_interpolate = None + if sensor.event_resolution == timedelta(0) and sensor.has_attribute("interpolate"): + mark_type = "area" + mark_interpolate = sensor.get_attribute("interpolate") chart_specs = { - "description": "A simple bar chart showing sensor data.", + "description": f"A simple {mark_type} chart showing sensor data.", # the sensor type is already shown as the y-axis title (avoid redundant info) "title": capitalize(sensor.name) if sensor.name != sensor.sensor_type else None, "layer": [ { "mark": { - "type": "bar", + "type": mark_type, + "interpolate": mark_interpolate, "clip": True, "width": {"band": 0.999}, }, diff --git a/flexmeasures/data/schemas/attributes.py b/flexmeasures/data/schemas/attributes.py new file mode 100644 index 000000000..fe9d3a92f --- /dev/null +++ b/flexmeasures/data/schemas/attributes.py @@ -0,0 +1,9 @@ +from typing import Any + +from altair.vegalite.schema import Interpolate + + +def validate_special_attributes(key: str, value: Any): + """Validate attributes with a special meaning in FlexMeasures.""" + if key == "interpolate": + Interpolate.validate(value)