Merge pull request #1211 from vantage6/release/4.4

Release/4.4.0

.github/workflows/release.yml

@@ -158,8 +158,8 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build support images
-        run: make support-image TAG=$version
+      - name: Build and push support images
+        run: make support-image TAG=$version PUSH_REG=true
 
   infrastructure-images:
     name: Build infrastructure images
@@ -195,21 +195,21 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - # we only release a new base image if it is a major or minor release
-        name: Build Docker base image
+        name: Build and push Docker base image
         if: ${{ env.patch == 0 }}
-        run: make base-image TAG=${major}.${minor}
+        run: make base-image TAG=${major}.${minor} PUSH_REG=true
 
       - # build the actual infrastructure images
-        name: Build Docker image
-        run: make image TAG=$version BASE=${major}.${minor}
+        name: Build and push Docker image
+        run: make image TAG=$version BASE=${major}.${minor} PUSH_REG=true
 
       - # build the algorithm store image
-        name: Build Docker algorithm store image
-        run: make algorithm-store-image TAG=$version BASE=${major}.${minor}
+        name: Build and push Docker algorithm store image
+        run: make algorithm-store-image TAG=$version BASE=${major}.${minor} PUSH_REG=true
 
       - # build the UI image
-        name: Build UI image
-        run: make ui-image TAG=$version BASE=${major}.${minor}
+        name: Build and push UI image
+        run: make ui-image TAG=$version BASE=${major}.${minor} PUSH_REG=true
 
   algorithm-base-image:
     name: Build algorithm base image
@@ -241,11 +241,11 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build Algorithm base image
-        run: make algorithm-base-image TAG=${major}.${minor}
+      - name: Build and push Algorithm base image
+        run: make algorithm-base-image TAG=${major}.${minor} PUSH_REG=true
 
-      - name: Build Algorithm OMOP base image
-        run: make algorithm-omop-base-image BASE=${major}.${minor} TAG=${major}.${minor}
+      - name: Build and push Algorithm OMOP base image
+        run: make algorithm-omop-base-image BASE=${major}.${minor} TAG=${major}.${minor} PUSH_REG=true
 
   # Update the server and node images on harbor2 for this major version,
   # including the 'live' tag that triggers a redeployment of the IKNL deployment

Makefile

@@ -5,10 +5,26 @@
 TAG ?= cotopaxi
 REGISTRY ?= harbor2.vantage6.ai
 PLATFORMS ?= linux/arm64,linux/amd64
+# Example for local development
+# TAG ?= local
+# REGISTRY ?= localhost
+# PLATFORMS ?= linux/amd64
 
 # infrastructure base image version
 BASE ?= 4.0
 
+# Use `make PUSH_REG=true` to push images to registry after building
+PUSH_REG ?= false
+
+
+# We use a conditional (true on any non-empty string) later. To avoid
+# accidents, we don't use user-controlled PUSH_REG directly.
+# See: https://www.gnu.org/software/make/manual/html_node/Conditional-Functions.html
+_condition_push :=
+ifeq ($(PUSH_REG), true)
+	_condition_push := not_empty_so_true
+endif
+
 help:
 	@echo "Available commands to 'make':"
 	@echo "  set-version          : set version (e.g set-version FLAGS=\"--version 2.0.0 --build 0 --spec alpha\")"
@@ -80,7 +96,7 @@ base-image:
 		--tag ${REGISTRY}/infrastructure/infrastructure-base:latest \
 		--platform ${PLATFORMS} \
 		-f ./docker/infrastructure-base.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 algorithm-base-image:
 	@echo "Building ${REGISTRY}/algorithms/algorithm-base:${TAG}"
@@ -90,7 +106,7 @@ algorithm-base-image:
 		--platform ${PLATFORMS} \
 		--build-arg TAG=${TAG} \
 		-f ./docker/algorithm-base.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 # FIXME FM 17-10-2023: This fails to build for arm64, this is because of
 # the r-base image.
@@ -103,7 +119,7 @@ algorithm-omop-base-image:
 		--build-arg TAG=${TAG} \
 		--platform linux/amd64 \
 		-f ./docker/algorithm-ohdsi-base.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 support-image:
 	@echo "Building support images"
@@ -121,7 +137,7 @@ support-squid-image:
 		--tag ${REGISTRY}/infrastructure/squid:latest \
 		--platform ${PLATFORMS} \
 		-f ./docker/squid.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 support-alpine-image:
 	@echo "Building ${REGISTRY}/infrastructure/alpine:${TAG}"
@@ -130,7 +146,7 @@ support-alpine-image:
 		--tag ${REGISTRY}/infrastructure/alpine:latest \
 		--platform ${PLATFORMS} \
 		-f ./docker/alpine.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 support-vpn-client-image:
 	@echo "Building ${REGISTRY}/infrastructure/vpn-client:${TAG}"
@@ -139,7 +155,7 @@ support-vpn-client-image:
 		--tag ${REGISTRY}/infrastructure/vpn-client:latest \
 		--platform ${PLATFORMS} \
 		-f ./docker/vpn-client.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 support-vpn-configurator-image:
 	@echo "Building ${REGISTRY}/infrastructure/vpn-configurator:${TAG}"
@@ -148,7 +164,7 @@ support-vpn-configurator-image:
 		--tag ${REGISTRY}/infrastructure/vpn-configurator:latest \
 		--platform ${PLATFORMS} \
 		-f ./docker/vpn-configurator.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 support-ssh-tunnel-image:
 	@echo "Building ${REGISTRY}/infrastructure/ssh-tunnel:${TAG}"
@@ -157,7 +173,7 @@ support-ssh-tunnel-image:
 		--tag ${REGISTRY}/infrastructure/ssh-tunnel:latest \
 		--platform ${PLATFORMS} \
 		-f ./docker/ssh-tunnel.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 image:
 	@echo "Building ${REGISTRY}/infrastructure/node:${TAG}"
@@ -169,7 +185,7 @@ image:
 		--build-arg BASE=${BASE} \
 		--platform ${PLATFORMS} \
 		-f ./docker/node-and-server.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 algorithm-store-image:
 	@echo "Building ${REGISTRY}/infrastructure/algorithm-store:${TAG}"
@@ -180,7 +196,7 @@ algorithm-store-image:
 		--build-arg BASE=${BASE} \
 		--platform ${PLATFORMS} \
 		-f ./docker/algorithm-store.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 ui-image:
 	@echo "Building ${REGISTRY}/infrastructure/ui:${TAG}"
@@ -191,7 +207,7 @@ ui-image:
 		--build-arg BASE=${BASE} \
 		--platform ${PLATFORMS} \
 		-f ./docker/ui.Dockerfile \
-		--push .
+		$(if ${_condition_push},--push .,.)
 
 rebuild:
 	@echo "------------------------------------"

docker/algorithm-store.Dockerfile

@@ -5,7 +5,7 @@
 # * harbor2.vantage6.ai/infrastructure/algorithm-store:x.x.x
 #
 ARG TAG=latest
-ARG BASE=4.3
+ARG BASE=4.4
 FROM harbor2.vantage6.ai/infrastructure/infrastructure-base:${BASE}
 
 LABEL version=${TAG}
@@ -36,6 +36,6 @@ RUN apt-get install --no-install-recommends --no-install-suggests -y \
 RUN CFLAGS="-I/usr/local/opt/openssl/include" \
   LDFLAGS="-L/usr/local/opt/openssl/lib" \
   UWSGI_PROFILE_OVERRIDE=ssl=true \
-  pip install uwsgi -Iv
+  pip install uwsgi==2.0.24 -Iv
 
 RUN chmod +x /vantage6/vantage6-algorithm-store/server.sh

docker/infrastructure-base.Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.10-slim-buster
 
-LABEL version="4.3"
+LABEL version="4.4"
 LABEL maintainer="Frank Martin <f.martin@iknl.nl>"
 
 # slim buster does not have gcc installed

docker/node-and-server.Dockerfile

@@ -6,7 +6,7 @@
 # * harbor2.vantage6.ai/infrastructure/server:x.x.x
 #
 ARG TAG=latest
-ARG BASE=4.3
+ARG BASE=4.4
 FROM harbor2.vantage6.ai/infrastructure/infrastructure-base:${BASE}
 
 LABEL version=${TAG}
@@ -47,7 +47,7 @@ RUN apt-get install --no-install-recommends --no-install-suggests -y \
 RUN CFLAGS="-I/usr/local/opt/openssl/include" \
   LDFLAGS="-L/usr/local/opt/openssl/lib" \
   UWSGI_PROFILE_OVERRIDE=ssl=true \
-  pip install uwsgi -Iv
+  pip install uwsgi==2.0.24 -Iv
 
 RUN chmod +x /vantage6/vantage6-server/server.sh
 

docker/ui.Dockerfile

@@ -1,6 +1,6 @@
 ARG TAG=latest
 ARG BASE=4.3
-FROM node:16 as node
+FROM node:20-alpine as node
 
 LABEL version=${TAG}
 LABEL maintainer="Bart van Beusekom <b.vanbeusekom@iknl.nl>, Frank Martin <f.martin@iknl.nl>"

docs/algorithms/develop.rst

@@ -97,20 +97,12 @@ as follows:
 
 .. code:: python
 
-   from vantage6.algorithm.tools.util import get_env_var
-
    def my_function():
-       input_file = get_env_var("INPUT_FILE")
-       token_file = get_env_var("DEFAULT_DATABASE_URI")
+       input_file = os.environ["INPUT_FILE"]
+       token_file = os.environ["DEFAULT_DATABASE_URI"]
 
        # do something with the input file and database URI
-
-.. note::
-
-   The ``get_env_var`` function is used here rather than the standard
-   ``os.environ`` dictionary because the environment variables are encoded
-   for security purposes. The ``get_env_var`` function will decode the
-   environment variable for you.
+       pass
 
 The environment variables that you specify in the node configuration file
 can be used in the exact same manner. You can view all environment variables

docs/release_notes.rst

@@ -1,6 +1,51 @@
 Release notes
 =============
 
+4.4.0
+-----
+
+*15 April 2024*
+
+- **Feature**
+
+ - Added visualization of a results table to the UI. The algorithm store is used to
+   store how the table should be visualized.
+   (`Issue#1057 <https://github.com/vantage6/vantage6/issues/1057>`_,
+   `PR#1195 <https://github.com/vantage6/vantage6/pull/1195>`_).
+ - Support for more types of algorithm arguments via the UI: lists of strings, ints,
+   floats and columns, and booleans
+   (`Issue#1119 <https://github.com/vantage6/vantage6/issues/1119>`_,
+   `PR#1190 <https://github.com/vantage6/vantage6/pull/1190>`_).
+ - Added configuration option to link algorithm stores to a server via the server
+   configuration (`PR#1156 <https://github.com/vantage6/vantage6/pull/1156>`_).
+ - Added a bunch of custom exceptions for algorithms to the algorithm tools
+   (`Issue#1185 <https://github.com/vantage6/vantage6/issues/1185>`_,
+   `PR#1205 <https://github.com/vantage6/vantage6/pull/1205>`_).
+ - Decoding the environment variables automatically in the algorithm wrapper, to prevent
+   that a user has to decode them manually
+   (`Issue#1056 <https://github.com/vantage6/vantage6/issues/1056>`_,
+   `PR#1197 <https://github.com/vantage6/vantage6/pull/1197>`_).
+ - Add option to delete roles in the UI
+   (`Issue#1113 <https://github.com/vantage6/vantage6/issues/1113>`_,
+   `PR#1199 <https://github.com/vantage6/vantage6/pull/1199>`_).
+ - Add option to register a node in the UI *after* creating/editing the collaboration
+   (`Issue#1122 <https://github.com/vantage6/vantage6/issues/1122>`_,
+   `PR#1202 <https://github.com/vantage6/vantage6/pull/1202>`_).
+
+- **Change**
+
+ - Updated idna dependency
+
+- **Bugfix**
+
+ - Do not mark algorithm runs as killed if they were completed before the user killed
+   the task to which the runs belong
+   (`Issue#1045 <https://github.com/vantage6/vantage6/issues/1045>`_,
+   `PR#1204 <https://github.com/vantage6/vantage6/pull/1204>`_).
+ - Fix UI code in a few places where pagination was not implemented properly
+   (`Issue#1126 <https://github.com/vantage6/vantage6/issues/1126>`_,
+   `PR#1203 <https://github.com/vantage6/vantage6/pull/1203>`_).
+
 4.3.4
 -----
 

docs/server/yaml/server_config.yaml

@@ -21,9 +21,10 @@ api_path: /api
 # Let the server know where it is hosted. This is used as a setting to
 # communicate back and forth with other vantage6 components such as the
 # algorithm store.
+# Example for the cotopaxi server
 server_url: https://cotopaxi.vantage6.ai
-# or if you are running the server locally with default settings:
-# server_url: http://localhost:5000/api
+# Example for running the server locally with default settings:
+# server_url: http://localhost:5000
 
 # The URI to the server database. This should be a valid SQLAlchemy URI,
 # e.g. for an Sqlite database: sqlite:///database-name.sqlite,
@@ -185,4 +186,23 @@ password_policy:
 #  port 3456 on localhost
 cors_allowed_origins:
   - https://portal.cotopaxi.vantage6.ai
-  - http://localhost:3456
+  - http://localhost:3456
+
+# set up which algorithm stores are available for all collaborations in the server.
+# In the example below, the vantage6 community algorithm store is made available to
+# this server. Note that the 'server_url' *has* to be set in the configuration file
+# for this to work (it is required to communicate with the algorithm store what the
+# server's address is).
+algorithm_stores:
+  # Each store should have a name and a URL.
+  - name: Community store
+    url: https://store.cotopaxi.vantage6.ai
+
+# development mode settings. Only use when running both the server and the algorithm
+# store that it communicates with locally
+dev:
+  # Specify the URI to the host. This is used to generate the correct URIs to
+  # communicate with the algorithm store. On Windows and Mac, you can use the special
+  # hostname `host.docker.internal` to refer to the host machine. On Linux, you
+  # should normally use http://172.17.0.1.
+  host_uri: http://host.docker.internal

requirements.txt

@@ -24,7 +24,7 @@ Flask-RESTful==0.3.10
 Flask-SocketIO==5.3.6
 gevent==23.9.1
 greenlet==2.0.2
-idna==3.4
+idna==3.7
 ipython==8.10.0
 isodate==0.6.1
 itsdangerous==2.1.2

vantage6-algorithm-store/requirements.txt

@@ -5,6 +5,7 @@ Flask-Principal==0.4.0
 flask-marshmallow==0.15.0
 Flask-RESTful==0.3.10
 gevent==23.9.1
+jsonschema==4.21.1
 marshmallow==3.19.0
 requests==2.31.0
 schema==0.7.5

vantage6-algorithm-store/setup.py

@@ -38,6 +38,7 @@
         "flask-marshmallow==0.15.0",
         "Flask-RESTful==0.3.10",
         "gevent==23.9.1",
+        "jsonschema==4.21.1",
         "marshmallow==3.19.0",
         "requests==2.31.0",
         "schema==0.7.5",

vantage6-algorithm-store/vantage6/algorithm/store/__init__.py

@@ -36,11 +36,12 @@
 from vantage6.common import logger_name
 from vantage6.common.globals import APPNAME
 from vantage6.backend.common.resource.output_schema import BaseHATEOASModelSchema
+from vantage6.backend.common.globals import HOST_URI_ENV
 
 # TODO move this to common, then remove dependency on CLI in algorithm store
 from vantage6.cli.context.algorithm_store import AlgorithmStoreContext
 from vantage6.algorithm.store._version import __version__
-from vantage6.algorithm.store.globals import API_PATH, HOST_URI_ENV
+from vantage6.algorithm.store.globals import API_PATH
 from vantage6.algorithm.store.globals import RESOURCES, SERVER_MODULE_NAME
 
 # TODO the following are simply copies of the same files in the server - refactor

vantage6-algorithm-store/vantage6/algorithm/store/_version.py

@@ -7,7 +7,7 @@
     __build__ = json.load(fp)
 
 # Module version
-version_info = (4, 3, 4, "final", __build__, 0)
+version_info = (4, 4, 0, "final", __build__, 0)
 
 # Module version stage suffix map
 _specifier_ = {"alpha": "a", "beta": "b", "candidate": "rc", "final": ""}