From 61d983a811873449b1dd716394d7cb37cdb73263 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 29 Nov 2020 22:27:33 +0100 Subject: [PATCH 01/74] ensure storage is in debug mode when in devel --- services/docker-compose.devel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index cb99877d3030..ea50460cdd02 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -187,3 +187,4 @@ services: - ../packages:/devel/packages environment: - SC_BOOT_MODE=debug-ptvsd + - STORAGE_LOGLEVEL=DEBUG From 13f3f2c64abcf42ed15db705183a3cb93a966e76 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 30 Nov 2020 13:02:17 +0100 Subject: [PATCH 02/74] format --- .../src/simcore_service_storage/dsm.py | 99 ++++++++++--------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/services/storage/src/simcore_service_storage/dsm.py b/services/storage/src/simcore_service_storage/dsm.py index 408166e7c443..7cb09580c36f 100644 --- a/services/storage/src/simcore_service_storage/dsm.py +++ b/services/storage/src/simcore_service_storage/dsm.py @@ -4,10 +4,10 @@ import re import shutil import tempfile +from collections import deque from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Dict, List, Optional, Tuple -from collections import deque import aiobotocore import aiofiles @@ -16,16 +16,14 @@ from aiohttp import web from aiopg.sa import Engine from blackfynn.base import UnauthorizedException -from sqlalchemy.sql import and_ -from tenacity import retry -from yarl import URL - from s3wrapper.s3_client import S3Client from servicelib.aiopg_utils import DBAPIError, PostgresRetryPolicyUponOperation from servicelib.client_session import get_client_session from servicelib.utils import fire_and_forget_task +from sqlalchemy.sql import and_ +from tenacity import retry +from yarl import URL -from .utils import expo from .datcore_wrapper import DatcoreWrapper from .models import ( DatasetMetaData, @@ -46,6 +44,7 @@ SIMCORE_S3_ID, SIMCORE_S3_STR, ) +from .utils import expo # pylint: disable=no-value-for-parameter # FIXME: E1120:No value for argument 'dml' in method call @@ -104,33 +103,33 @@ def to_tuple(self): @attr.s(auto_attribs=True) class DataStorageManager: - """ Data storage manager + """Data storage manager - The dsm has access to the database for all meta data and to the actual backend. For now this - is simcore's S3 [minio] and the datcore storage facilities. + The dsm has access to the database for all meta data and to the actual backend. For now this + is simcore's S3 [minio] and the datcore storage facilities. - For all data that is in-house (simcore.s3, ...) we keep a synchronized database with meta information - for the physical files. + For all data that is in-house (simcore.s3, ...) we keep a synchronized database with meta information + for the physical files. - For physical changes on S3, that might be time-consuming, the db keeps a state (delete and upload mostly) + For physical changes on S3, that might be time-consuming, the db keeps a state (delete and upload mostly) - The dsm provides the following additional functionalities: + The dsm provides the following additional functionalities: - - listing of folders for a given users, optionally filtered using a regular expression and optionally - sorted by one of the meta data keys + - listing of folders for a given users, optionally filtered using a regular expression and optionally + sorted by one of the meta data keys - - upload/download of files + - upload/download of files - client -> S3 : presigned upload link - S3 -> client : presigned download link - datcore -> client: presigned download link - S3 -> datcore: local copy and then upload via their api + client -> S3 : presigned upload link + S3 -> client : presigned download link + datcore -> client: presigned download link + S3 -> datcore: local copy and then upload via their api - minio/S3 and postgres can talk nicely with each other via Notifications using rabbigMQ which we already have. - See: + minio/S3 and postgres can talk nicely with each other via Notifications using rabbigMQ which we already have. + See: - https://blog.minio.io/part-5-5-publish-minio-events-via-postgresql-50f6cc7a7346 - https://docs.minio.io/docs/minio-bucket-notification-guide.html + https://blog.minio.io/part-5-5-publish-minio-events-via-postgresql-50f6cc7a7346 + https://docs.minio.io/docs/minio-bucket-notification-guide.html """ s3_client: S3Client @@ -166,7 +165,7 @@ def location_from_id(cls, location_id: str): return _location_from_id(location_id) async def ping_datcore(self, user_id: str) -> bool: - """ Checks whether user account in datcore is accesible + """Checks whether user account in datcore is accesible :param user_id: user identifier :type user_id: str @@ -193,13 +192,13 @@ async def ping_datcore(self, user_id: str) -> bool: async def list_files( self, user_id: str, location: str, uuid_filter: str = "", regex: str = "" ) -> FileMetaDataExVec: - """ Returns a list of file paths + """Returns a list of file paths - Works for simcore.s3 and datcore + Works for simcore.s3 and datcore - Can filter on uuid: useful to filter on project_id/node_id + Can filter on uuid: useful to filter on project_id/node_id - Can filter upon regular expression (for now only on key: value pairs of the FileMetaData) + Can filter upon regular expression (for now only on key: value pairs of the FileMetaData) """ data = deque() if location == SIMCORE_S3_STR: @@ -308,9 +307,9 @@ async def list_files_dataset( return data async def list_datasets(self, user_id: str, location: str) -> DatasetMetaDataVec: - """ Returns a list of top level datasets + """Returns a list of top level datasets - Works for simcore.s3 and datcore + Works for simcore.s3 and datcore """ data = [] @@ -363,15 +362,15 @@ async def list_file( return data async def delete_file(self, user_id: str, location: str, file_uuid: str): - """ Deletes a file given its fmd and location + """Deletes a file given its fmd and location - Additionally requires a user_id for 3rd party auth + Additionally requires a user_id for 3rd party auth - For internal storage, the db state should be updated upon completion via - Notification mechanism + For internal storage, the db state should be updated upon completion via + Notification mechanism - For simcore.s3 we can use the file_name - For datcore we need the full path + For simcore.s3 we can use the file_name + For datcore we need the full path """ if location == SIMCORE_S3_STR: to_delete = [] @@ -462,13 +461,15 @@ async def metadata_file_updater( await asyncio.sleep(sleep_amount) continue + file_e_tag = result["Contents"][0]["ETag"] # finally update the data in the database and exit continue_loop = False logger.info( - "Obtained this from S3: new_file_size=%s new_last_modified=%s", + "Obtained this from S3: new_file_size=%s new_last_modified=%s file ETag=%s", new_file_size, new_last_modified, + file_e_tag, ) async with self.engine.acquire() as conn: @@ -635,22 +636,22 @@ async def download_link_datcore(self, user_id: str, file_id: str) -> Dict[str, s async def deep_copy_project_simcore_s3( self, user_id: str, source_project, destination_project, node_mapping ): - """ Parses a given source project and copies all related files to the destination project + """Parses a given source project and copies all related files to the destination project - Since all files are organized as + Since all files are organized as - project_id/node_id/filename or links to datcore + project_id/node_id/filename or links to datcore - this function creates a new folder structure + this function creates a new folder structure - project_id/node_id/filename + project_id/node_id/filename - and copies all files to the corresponding places. + and copies all files to the corresponding places. - Additionally, all external files from datcore are being copied and the paths in the destination - project are adapted accordingly + Additionally, all external files from datcore are being copied and the paths in the destination + project are adapted accordingly - Lastly, the meta data db is kept in sync + Lastly, the meta data db is kept in sync """ source_folder = source_project["uuid"] dest_folder = destination_project["uuid"] @@ -773,8 +774,8 @@ async def deep_copy_project_simcore_s3( async def delete_project_simcore_s3( self, user_id: str, project_id: str, node_id: Optional[str] = None ) -> web.Response: - """ Deletes all files from a given node in a project in simcore.s3 and updated db accordingly. - If node_id is not given, then all the project files db entries are deleted. + """Deletes all files from a given node in a project in simcore.s3 and updated db accordingly. + If node_id is not given, then all the project files db entries are deleted. """ async with self.engine.acquire() as conn: From e624fc2944d2c88cf75485d063635b9d713ad3c6 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 30 Nov 2020 15:12:31 +0100 Subject: [PATCH 03/74] missing alembic --- packages/simcore-sdk/requirements/_test.in | 1 + packages/simcore-sdk/requirements/_test.txt | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index 2abd0641c010..e85a9e60065b 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -8,6 +8,7 @@ -c _base.txt # testing +alembic coverage pytest pytest-aiohttp # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76 diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index 09bbd727d2a7..7ca15a80130b 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -6,6 +6,7 @@ # aiohttp==3.7.3 # via -c requirements/_base.txt, aioresponses, pytest-aiohttp aioresponses==0.7.1 # via -r requirements/_test.in +alembic==1.4.3 # via -r requirements/_test.in apipkg==1.5 # via execnet astroid==2.4.2 # via pylint async-timeout==3.0.1 # via -c requirements/_base.txt, aiohttp @@ -29,6 +30,7 @@ multidict==5.1.0 # via -c requirements/_base.txt, aiohttp, yarl packaging==20.7 # via pytest, pytest-sugar pluggy==0.13.1 # via pytest pprintpp==0.4.0 # via pytest-icdiff +psycopg2-binary==2.8.6 # via -c requirements/_base.txt, sqlalchemy py==1.9.0 # via pytest, pytest-forked pylint==2.6.0 # via -r requirements/_test.in pyparsing==2.4.7 # via packaging @@ -42,9 +44,12 @@ pytest-runner==5.2 # via -r requirements/_test.in pytest-sugar==0.9.4 # via -r requirements/_test.in pytest-xdist==2.1.0 # via -r requirements/_test.in pytest==6.1.2 # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-forked, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar, pytest-xdist +python-dateutil==2.8.1 # via alembic python-dotenv==0.15.0 # via -r requirements/_test.in +python-editor==1.0.4 # via alembic requests==2.25.0 # via -r requirements/_test.in, coveralls, docker -six==1.15.0 # via -c requirements/_base.txt, astroid, docker, websocket-client +six==1.15.0 # via -c requirements/_base.txt, astroid, docker, packaging, python-dateutil, websocket-client +sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -c requirements/_base.txt, alembic termcolor==1.1.0 # via pytest-sugar toml==0.10.2 # via pylint, pytest typed-ast==1.4.1 # via astroid From 4410e61c15d5cc0fd38827b07b71f695c8f630b3 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 30 Nov 2020 15:22:21 +0100 Subject: [PATCH 04/74] added pytest-black --- packages/simcore-sdk/requirements/_test.in | 1 + packages/simcore-sdk/requirements/_test.txt | 6 +++++- packages/simcore-sdk/requirements/_tools.txt | 10 +++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index e85a9e60065b..76d6c3325a82 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -12,6 +12,7 @@ alembic coverage pytest pytest-aiohttp # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76 +pytest-black pytest-cov pytest-icdiff pytest-instafail diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index 7ca15a80130b..adf1c4a1c2ec 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -8,11 +8,13 @@ aiohttp==3.7.3 # via -c requirements/_base.txt, aioresponses, pytest- aioresponses==0.7.1 # via -r requirements/_test.in alembic==1.4.3 # via -r requirements/_test.in apipkg==1.5 # via execnet +appdirs==1.4.4 # via black astroid==2.4.2 # via pylint async-timeout==3.0.1 # via -c requirements/_base.txt, aiohttp attrs==20.3.0 # via -c requirements/_base.txt, aiohttp, pytest certifi==2020.12.5 # via -c requirements/_base.txt, requests chardet==3.0.4 # via -c requirements/_base.txt, aiohttp, requests +click==7.1.2 # via black coverage==5.3 # via -r requirements/_test.in, coveralls, pytest-cov coveralls==2.2.0 # via -r requirements/_test.in docker==4.4.0 # via -r requirements/_test.in @@ -35,6 +37,7 @@ py==1.9.0 # via pytest, pytest-forked pylint==2.6.0 # via -r requirements/_test.in pyparsing==2.4.7 # via packaging pytest-aiohttp==0.3.0 # via -r requirements/_test.in +pytest-black==0.3.12 # via -r requirements/_test.in pytest-cov==2.10.1 # via -r requirements/_test.in pytest-forked==1.3.0 # via pytest-xdist pytest-icdiff==0.5 # via -r requirements/_test.in @@ -43,10 +46,11 @@ pytest-mock==3.3.1 # via -r requirements/_test.in pytest-runner==5.2 # via -r requirements/_test.in pytest-sugar==0.9.4 # via -r requirements/_test.in pytest-xdist==2.1.0 # via -r requirements/_test.in -pytest==6.1.2 # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-forked, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar, pytest-xdist +pytest==6.1.2 # via -r requirements/_test.in, pytest-aiohttp, pytest-black, pytest-cov, pytest-forked, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar, pytest-xdist python-dateutil==2.8.1 # via alembic python-dotenv==0.15.0 # via -r requirements/_test.in python-editor==1.0.4 # via alembic +regex==2020.11.13 # via black requests==2.25.0 # via -r requirements/_test.in, coveralls, docker six==1.15.0 # via -c requirements/_base.txt, astroid, docker, packaging, python-dateutil, websocket-client sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -c requirements/_base.txt, alembic diff --git a/packages/simcore-sdk/requirements/_tools.txt b/packages/simcore-sdk/requirements/_tools.txt index 17b080dc9f0f..ea5a8ac2953a 100644 --- a/packages/simcore-sdk/requirements/_tools.txt +++ b/packages/simcore-sdk/requirements/_tools.txt @@ -4,8 +4,8 @@ # # pip-compile --output-file=requirements/_tools.txt requirements/_tools.in # -appdirs==1.4.4 # via black, virtualenv -black==20.8b1 # via -r requirements/../../../requirements/devenv.txt +appdirs==1.4.4 # via -c requirements/_test.txt, black, virtualenv +black==20.8b1 # via -c requirements/_test.txt, -r requirements/../../../requirements/devenv.txt bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt cfgv==3.2.0 # via pre-commit click==7.1.2 # via black, pip-tools @@ -16,13 +16,13 @@ identify==1.5.10 # via pre-commit importlib-metadata==3.1.1 # via -c requirements/_base.txt, -c requirements/_test.txt, pre-commit, virtualenv importlib-resources==3.3.0 # via pre-commit, virtualenv isort==5.6.4 # via -c requirements/_test.txt, -r requirements/../../../requirements/devenv.txt -mypy-extensions==0.4.3 # via black +mypy-extensions==0.4.3 # via -c requirements/_test.txt, black nodeenv==1.5.0 # via pre-commit -pathspec==0.8.1 # via black +pathspec==0.8.1 # via -c requirements/_test.txt, black pip-tools==5.4.0 # via -r requirements/../../../requirements/devenv.txt pre-commit==2.9.3 # via -r requirements/../../../requirements/devenv.txt pyyaml==5.3.1 # via -c requirements/_base.txt, pre-commit -regex==2020.11.13 # via black +regex==2020.11.13 # via -c requirements/_test.txt, black six==1.15.0 # via -c requirements/_base.txt, -c requirements/_test.txt, pip-tools, virtualenv toml==0.10.2 # via -c requirements/_test.txt, black, pre-commit typed-ast==1.4.1 # via -c requirements/_test.txt, black From f5e09e38f13574e2fe5ede4fea57c6157c788276 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 30 Nov 2020 15:22:46 +0100 Subject: [PATCH 05/74] formatting --- .../tests/unit/test_schema_item.py | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/packages/simcore-sdk/tests/unit/test_schema_item.py b/packages/simcore-sdk/tests/unit/test_schema_item.py index 24bc48dd6e31..0701670d209a 100644 --- a/packages/simcore-sdk/tests/unit/test_schema_item.py +++ b/packages/simcore-sdk/tests/unit/test_schema_item.py @@ -2,17 +2,24 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import pytest from copy import deepcopy -from simcore_sdk.node_ports import exceptions, config + +import pytest +from simcore_sdk.node_ports import config, exceptions from simcore_sdk.node_ports._schema_item import SchemaItem + def test_default_item(): with pytest.raises(exceptions.InvalidProtocolError): - item = SchemaItem() #pylint: disable=W0612 + item = SchemaItem() # pylint: disable=W0612 -def test_check_item_required_fields(): #pylint: disable=W0612 - required_parameters = {key:"defaultValue" for key, required in config.SCHEMA_ITEM_KEYS.items() if required} + +def test_check_item_required_fields(): # pylint: disable=W0612 + required_parameters = { + key: "defaultValue" + for key, required in config.SCHEMA_ITEM_KEYS.items() + if required + } # this shall not trigger an exception SchemaItem(**required_parameters) @@ -22,8 +29,15 @@ def test_check_item_required_fields(): #pylint: disable=W0612 with pytest.raises(exceptions.InvalidProtocolError): SchemaItem(**parameters) + def test_item_construction_default(): - item = SchemaItem(key="a key", label="a label", description="a description", type="a type", displayOrder=2) + item = SchemaItem( + key="a key", + label="a label", + description="a description", + type="a type", + displayOrder=2, + ) assert item.key == "a key" assert item.label == "a label" assert item.description == "a description" @@ -33,13 +47,23 @@ def test_item_construction_default(): assert item.defaultValue == None assert item.widget == None + def test_item_construction_with_optional_params(): - item = SchemaItem(key="a key", label="a label", description="a description", type="a type", displayOrder=2, fileToKeyMap={"file1.txt":"a key"}, defaultValue="some value", widget={}) + item = SchemaItem( + key="a key", + label="a label", + description="a description", + type="a type", + displayOrder=2, + fileToKeyMap={"file1.txt": "a key"}, + defaultValue="some value", + widget={}, + ) assert item.key == "a key" assert item.label == "a label" assert item.description == "a description" assert item.type == "a type" assert item.displayOrder == 2 - assert item.fileToKeyMap == {"file1.txt":"a key"} + assert item.fileToKeyMap == {"file1.txt": "a key"} assert item.defaultValue == "some value" assert item.widget == {} From 266e7711bc9da8a289fae75053de2d6de360366e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 30 Nov 2020 17:54:31 +0100 Subject: [PATCH 06/74] added annotations autoformating --- .pre-commit-config.yaml | 2 +- .../simcore-sdk/src/simcore_sdk/config/db.py | 30 ++--- .../src/simcore_sdk/models/__init__.py | 7 +- .../src/simcore_sdk/models/base.py | 4 +- .../src/simcore_sdk/node_data/__init__.py | 2 +- .../src/simcore_sdk/node_ports/__init__.py | 1 - .../src/simcore_sdk/node_ports/_data_item.py | 11 +- .../src/simcore_sdk/node_ports/_item.py | 19 +-- .../simcore_sdk/node_ports/_schema_item.py | 5 +- .../src/simcore_sdk/node_ports/dbmanager.py | 5 +- .../src/simcore_sdk/node_ports/filemanager.py | 6 +- .../src/simcore_sdk/node_ports/nodeports.py | 4 +- .../simcore_sdk/node_ports/serialization.py | 9 +- .../simcore-sdk/tests/helpers/utils_docker.py | 70 +++++++---- .../tests/helpers/utils_environs.py | 76 ++++++----- .../simcore-sdk/tests/integration/conftest.py | 70 ++++++----- .../tests/integration/np_helpers.py | 54 ++++---- .../tests/integration/test_dbmanager.py | 12 +- .../tests/integration/test_filemanager.py | 27 +++- .../tests/integration/test_nodeports.py | 118 ++++++++++-------- .../simcore-sdk/tests/unit/test_data_item.py | 4 +- .../tests/unit/test_data_manager.py | 24 ++-- packages/simcore-sdk/tests/unit/test_item.py | 21 ++-- .../simcore-sdk/tests/unit/test_itemstlist.py | 7 +- 24 files changed, 353 insertions(+), 235 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 26605c9c3fee..ebb4d2af5781 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks -exclude: "^.venv$|^.cache$|^.pytest_cache$" +exclude: "^.venv$|^.cache$|^.pytest_cache$|^.*/_item.py$" default_language_version: python: python3.6 repos: diff --git a/packages/simcore-sdk/src/simcore_sdk/config/db.py b/packages/simcore-sdk/src/simcore_sdk/config/db.py index 26b6d2206cf1..a67603685b8a 100644 --- a/packages/simcore-sdk/src/simcore_sdk/config/db.py +++ b/packages/simcore-sdk/src/simcore_sdk/config/db.py @@ -5,21 +5,22 @@ import trafaret as T -CONFIG_SCHEMA = T.Dict({ - "database": T.String(), - "user": T.String(), - "password": T.String(), - T.Key("minsize", default=1 ,optional=True): T.ToInt(), - T.Key("maxsize", default=4, optional=True): T.ToInt(), - "host": T.Or( T.String, T.Null), - "port": T.Or( T.ToInt, T.Null), - "endpoint": T.Or( T.String, T.Null) -}) +CONFIG_SCHEMA = T.Dict( + { + "database": T.String(), + "user": T.String(), + "password": T.String(), + T.Key("minsize", default=1, optional=True): T.ToInt(), + T.Key("maxsize", default=4, optional=True): T.ToInt(), + "host": T.Or(T.String, T.Null), + "port": T.Or(T.ToInt, T.Null), + "endpoint": T.Or(T.String, T.Null), + } +) # TODO: deprecate! -class Config(): - +class Config: def __init__(self): # TODO: uniform config classes . see server.config file POSTGRES_URL = env.get("POSTGRES_ENDPOINT", "postgres:5432") @@ -31,8 +32,9 @@ def __init__(self): self._pwd = POSTGRES_PW self._url = POSTGRES_URL self._db = POSTGRES_DB - self._endpoint = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}'.format( - user=self._user, pw=self._pwd, url=self._url, db=self._db) + self._endpoint = "postgresql+psycopg2://{user}:{pw}@{url}/{db}".format( + user=self._user, pw=self._pwd, url=self._url, db=self._db + ) @property def endpoint(self): diff --git a/packages/simcore-sdk/src/simcore_sdk/models/__init__.py b/packages/simcore-sdk/src/simcore_sdk/models/__init__.py index a2dcfc5df6cd..06f3450664c1 100644 --- a/packages/simcore-sdk/src/simcore_sdk/models/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/models/__init__.py @@ -1,8 +1,7 @@ from . import pipeline_models +from .base import metadata + # Add here new models -from .base import metadata -__all__ = ( - 'metadata' -) \ No newline at end of file +__all__ = "metadata" diff --git a/packages/simcore-sdk/src/simcore_sdk/models/base.py b/packages/simcore-sdk/src/simcore_sdk/models/base.py index 2431525cfa3f..72ba04fee4b4 100644 --- a/packages/simcore-sdk/src/simcore_sdk/models/base.py +++ b/packages/simcore-sdk/src/simcore_sdk/models/base.py @@ -1,5 +1,3 @@ from simcore_postgres_database.models.base import metadata -__all__ = [ - "metadata" -] +__all__ = ["metadata"] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py index 8fbf7dfbfe2a..595d6f1ed7c6 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py @@ -1 +1 @@ -from . import data_manager \ No newline at end of file +from . import data_manager diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py index 8c9141e1dae4..9a2fd0a3f2b4 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py @@ -9,4 +9,3 @@ # in that sense it should not log stuff unless the application code wants it to be so. log = logging.getLogger(__name__) log.addHandler(logging.NullHandler()) - diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_item.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_item.py index 906a8bd8b1cb..6cd34cce2916 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_item.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_item.py @@ -2,23 +2,28 @@ import collections import logging +from typing import Dict, Optional, Union from . import config, exceptions log = logging.getLogger(__name__) +DataItemValue = Optional[Union[int, float, bool, str, Dict[str, Union[int, str]]]] + _DataItem = collections.namedtuple("_DataItem", config.DATA_ITEM_KEYS.keys()) + class DataItem(_DataItem): - """Encapsulates a Data Item and provides accessors functions + """Encapsulates a Data Item and provides accessors functions""" - """ def __new__(cls, **kwargs): new_kargs = dict.fromkeys(config.DATA_ITEM_KEYS.keys()) for key, required in config.DATA_ITEM_KEYS.items(): if key not in kwargs: if required: - raise exceptions.InvalidProtocolError(kwargs, "key \"%s\" is missing" % (str(key))) + raise exceptions.InvalidProtocolError( + kwargs, 'key "%s" is missing' % (str(key)) + ) new_kargs[key] = None else: new_kargs[key] = kwargs[key] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py index 6bb3aed15ebe..0119508e795c 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py @@ -6,13 +6,18 @@ from yarl import URL from . import config, data_items_utils, exceptions, filemanager -from ._data_item import DataItem +from ._data_item import DataItem, DataItemValue from ._schema_item import SchemaItem log = logging.getLogger(__name__) -def _check_type(item_type: str, value: Union[int, float, bool, str, Dict]): +ItemConcreteValue = Union[int, float, bool, str, Path] +NodeLink = Dict[str,str] +StoreLink = Dict[str,str] +DownloadLink = Dict[str,str] + +def _check_type(item_type: str, value: DataItemValue): if item_type not in config.TYPE_TO_PYTHON_TYPE_MAP and not data_items_utils.is_file_type(item_type): raise exceptions.InvalidItemTypeError(item_type, value) @@ -74,7 +79,7 @@ def __getattr__(self, name: str): raise AttributeError - async def get(self) -> Union[int, float, bool, str, Path]: + async def get(self) -> ItemConcreteValue: """ gets data converted to the underlying type :raises exceptions.InvalidProtocolError: if the underlying type is unknown @@ -124,7 +129,7 @@ async def get(self) -> Union[int, float, bool, str, Path]: config.TYPE_TO_PYTHON_TYPE_MAP[self.type]["converter"](self.value) ) - async def set(self, value: Union[int, float, bool, str, Path]): + async def set(self, value: ItemConcreteValue): """ sets the data to the underlying port :param value: must be convertible to a string, or an exception will be thrown. @@ -171,7 +176,7 @@ async def set(self, value: Union[int, float, bool, str, Path]): log.debug("database updated") async def __get_value_from_link( - self, value: Dict[str, str] + self, value: NodeLink ) -> Union[int, float, bool, str, Path]: # pylint: disable=R1710 log.debug("Getting value %s", value) node_uuid, port_key = data_items_utils.decode_link(value) @@ -187,7 +192,7 @@ async def __get_value_from_link( log.debug("Received node from DB %s, now returning value", other_nodeports) return await other_nodeports.get(port_key) - async def __get_value_from_store(self, value: Dict[str, str]) -> Path: + async def __get_value_from_store(self, value: StoreLink) -> Path: log.debug("Getting value from storage %s", value) store_id, s3_path = data_items_utils.decode_store(value) # do not make any assumption about s3_path, it is a str containing stuff that can be anything depending on the store @@ -206,7 +211,7 @@ async def __get_value_from_store(self, value: Dict[str, str]) -> Path: return downloaded_file - async def _get_value_from_download_link(self, value: Dict[str,str]) -> Path: + async def _get_value_from_download_link(self, value: DownloadLink) -> Path: log.debug("Getting value from download link [%s] with label %s", value["downloadLink"], value.get("label", "undef")) download_link = URL(value["downloadLink"]) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_item.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_item.py index 87ed6a51a47d..b527169eda7a 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_item.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_item.py @@ -7,13 +7,16 @@ _SchemaItem = collections.namedtuple("_SchemaItem", config.SCHEMA_ITEM_KEYS.keys()) + class SchemaItem(_SchemaItem): def __new__(cls, **kwargs): new_kargs = dict.fromkeys(config.SCHEMA_ITEM_KEYS.keys()) for key, required in config.SCHEMA_ITEM_KEYS.items(): if key not in kwargs: if required: - raise exceptions.InvalidProtocolError(kwargs, "key \"%s\" is missing" % (str(key))) + raise exceptions.InvalidProtocolError( + kwargs, 'key "%s" is missing' % (str(key)) + ) # new_kargs[key] = None else: new_kargs[key] = kwargs[key] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py index de9202e30fca..f038e2d5832e 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py @@ -5,8 +5,6 @@ import aiopg.sa import tenacity -from sqlalchemy import and_ - from servicelib.aiopg_utils import ( DataSourceName, PostgresRetryPolicyUponInitialization, @@ -14,9 +12,10 @@ is_postgres_responsive, ) from simcore_postgres_database.models.comp_tasks import comp_tasks -from .exceptions import NodeNotFound +from sqlalchemy import and_ from . import config +from .exceptions import NodeNotFound log = logging.getLogger(__name__) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py index 0512fa1f47e9..c1b5ab895316 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py @@ -5,16 +5,16 @@ from pathlib import Path from typing import Optional +import aiofiles from aiohttp import ClientSession, ClientTimeout from yarl import URL -import aiofiles +from models_library.settings.services_common import ServicesCommonSettings from simcore_service_storage_sdk import ApiClient, Configuration, UsersApi from simcore_service_storage_sdk.rest import ApiException -from models_library.settings.services_common import ServicesCommonSettings -from . import config, exceptions from ..config.http_clients import client_request_settings +from . import config, exceptions log = logging.getLogger(__name__) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py index 5cc21371c5f9..56ad267dafea 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py @@ -19,9 +19,7 @@ class Nodeports: - """Allows the client to access the inputs and outputs assigned to the node - - """ + """Allows the client to access the inputs and outputs assigned to the node""" _version = "0.1" diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py index 1033e9e730f5..a45cbbf16b55 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py @@ -19,7 +19,7 @@ async def create_from_json( db_mgr: DBManager, auto_read: bool = False, auto_write: bool = False ): - """ creates a Nodeports object provided a json configuration in form of a callback function + """creates a Nodeports object provided a json configuration in form of a callback function :param db_mgr: interface object to connect to nodeports description :param auto_read: the nodeports object shall automatically update itself when set to True, defaults to False @@ -64,7 +64,7 @@ async def create_nodeports_from_uuid(db_mgr: DBManager, node_uuid: str): async def save_to_json(nodeports_obj) -> None: - """ Encodes a Nodeports object to json and calls a linked writer if available. + """Encodes a Nodeports object to json and calls a linked writer if available. :param nodeports_obj: the object to encode :type nodeports_obj: Nodeports @@ -94,7 +94,10 @@ def default(self, o): # pylint: disable=E0202 return { # pylint: disable=W0212 "version": o._version, - "schema": {"inputs": o._input_schemas, "outputs": o._output_schemas,}, + "schema": { + "inputs": o._input_schemas, + "outputs": o._output_schemas, + }, "inputs": o._inputs_payloads, "outputs": o._outputs_payloads, } diff --git a/packages/simcore-sdk/tests/helpers/utils_docker.py b/packages/simcore-sdk/tests/helpers/utils_docker.py index 2b0bcb4de339..e5cc50b62071 100644 --- a/packages/simcore-sdk/tests/helpers/utils_docker.py +++ b/packages/simcore-sdk/tests/helpers/utils_docker.py @@ -1,4 +1,3 @@ - import logging import os import subprocess @@ -12,38 +11,50 @@ log = logging.getLogger(__name__) + @retry( - wait=wait_fixed(2), - stop=stop_after_attempt(10), - after=after_log(log, logging.WARN)) -def get_service_published_port(service_name: str, target_port: Optional[int]=None) -> str: + wait=wait_fixed(2), stop=stop_after_attempt(10), after=after_log(log, logging.WARN) +) +def get_service_published_port( + service_name: str, target_port: Optional[int] = None +) -> str: """ - WARNING: ENSURE that service name exposes a port in Dockerfile file or docker-compose config file + WARNING: ENSURE that service name exposes a port in Dockerfile file or docker-compose config file """ # NOTE: retries since services can take some time to start client = docker.from_env() services = [x for x in client.services.list() if service_name in x.name] if not services: - raise RuntimeError(f"Cannot find published port for service '{service_name}'. Probably services still not started.") + raise RuntimeError( + f"Cannot find published port for service '{service_name}'. Probably services still not started." + ) service_ports = services[0].attrs["Endpoint"].get("Ports") if not service_ports: - raise RuntimeError(f"Cannot find published port for service '{service_name}' in endpoint. Probably services still not started.") + raise RuntimeError( + f"Cannot find published port for service '{service_name}' in endpoint. Probably services still not started." + ) published_port = None - msg = ", ".join( f"{p.get('TargetPort')} -> {p.get('PublishedPort')}" for p in service_ports ) + msg = ", ".join( + f"{p.get('TargetPort')} -> {p.get('PublishedPort')}" for p in service_ports + ) if target_port is None: - if len(service_ports)>1: - log.warning("Multiple ports published in service '%s': %s. Defaulting to first", service_name, msg) + if len(service_ports) > 1: + log.warning( + "Multiple ports published in service '%s': %s. Defaulting to first", + service_name, + msg, + ) published_port = service_ports[0]["PublishedPort"] else: target_port = int(target_port) for p in service_ports: - if p['TargetPort'] == target_port: - published_port = p['PublishedPort'] + if p["TargetPort"] == target_port: + published_port = p["PublishedPort"] break if published_port is None: @@ -55,28 +66,37 @@ def get_service_published_port(service_name: str, target_port: Optional[int]=Non def run_docker_compose_config( docker_compose_paths: Union[List[Path], Path], workdir: Path, - destination_path: Optional[Path]=None) -> Dict: - """ Runs docker-compose config to validate and resolve a compose file configuration + destination_path: Optional[Path] = None, +) -> Dict: + """Runs docker-compose config to validate and resolve a compose file configuration - - Composes all configurations passed in 'docker_compose_paths' - - Takes 'workdir' as current working directory (i.e. all '.env' files there will be captured) - - Saves resolved output config to 'destination_path' (if given) + - Composes all configurations passed in 'docker_compose_paths' + - Takes 'workdir' as current working directory (i.e. all '.env' files there will be captured) + - Saves resolved output config to 'destination_path' (if given) """ if not isinstance(docker_compose_paths, List): - docker_compose_paths = [docker_compose_paths, ] + docker_compose_paths = [ + docker_compose_paths, + ] temp_dir = None if destination_path is None: - temp_dir = Path(tempfile.mkdtemp(prefix='')) - destination_path = temp_dir / 'docker-compose.yml' + temp_dir = Path(tempfile.mkdtemp(prefix="")) + destination_path = temp_dir / "docker-compose.yml" - config_paths = [ f"-f {os.path.relpath(docker_compose_path, workdir)}" for docker_compose_path in docker_compose_paths] + config_paths = [ + f"-f {os.path.relpath(docker_compose_path, workdir)}" + for docker_compose_path in docker_compose_paths + ] configs_prefix = " ".join(config_paths) - subprocess.run( f"docker-compose {configs_prefix} config > {destination_path}", - shell=True, check=True, - cwd=workdir) + subprocess.run( + f"docker-compose {configs_prefix} config > {destination_path}", + shell=True, + check=True, + cwd=workdir, + ) with destination_path.open() as f: config = yaml.safe_load(f) diff --git a/packages/simcore-sdk/tests/helpers/utils_environs.py b/packages/simcore-sdk/tests/helpers/utils_environs.py index 006245ef43be..40a54b5206df 100644 --- a/packages/simcore-sdk/tests/helpers/utils_environs.py +++ b/packages/simcore-sdk/tests/helpers/utils_environs.py @@ -8,15 +8,16 @@ import yaml -VARIABLE_SUBSTITUTION = re.compile(r'\$\{(\w+)+') # +VARIABLE_SUBSTITUTION = re.compile(r"\$\{(\w+)+") # + def load_env(file_handler) -> Dict: - """ Deserializes an environment file like .env-devel and - returns a key-value map of the environment + """Deserializes an environment file like .env-devel and + returns a key-value map of the environment - Analogous to json.load + Analogous to json.load """ - PATTERN_ENVIRON_EQUAL= re.compile(r"^(\w+)=(.*)$") + PATTERN_ENVIRON_EQUAL = re.compile(r"^(\w+)=(.*)$") # Works even for `POSTGRES_EXPORTER_DATA_SOURCE_NAME=postgresql://simcore:simcore@postgres:5432/simcoredb?sslmode=disable` environ = {} @@ -27,30 +28,41 @@ def load_env(file_handler) -> Dict: environ[key] = str(value) return environ -def eval_environs_in_docker_compose(docker_compose: Dict, docker_compose_dir: Path, - host_environ: Dict=None, *, use_env_devel=True): - """ Resolves environments in docker compose and sets them under 'environment' section - TODO: deprecated. Use instead docker-compose config in services/web/server/tests/integration/fixtures/docker_compose.py - SEE https://docs.docker.com/compose/environment-variables/ +def eval_environs_in_docker_compose( + docker_compose: Dict, + docker_compose_dir: Path, + host_environ: Dict = None, + *, + use_env_devel=True +): + """Resolves environments in docker compose and sets them under 'environment' section + + TODO: deprecated. Use instead docker-compose config in services/web/server/tests/integration/fixtures/docker_compose.py + SEE https://docs.docker.com/compose/environment-variables/ """ content = deepcopy(docker_compose) for _name, service in content["services"].items(): - replace_environs_in_docker_compose_service(service, docker_compose_dir, - host_environ, use_env_devel=use_env_devel) + replace_environs_in_docker_compose_service( + service, docker_compose_dir, host_environ, use_env_devel=use_env_devel + ) return content -def replace_environs_in_docker_compose_service(service_section: Dict, + +def replace_environs_in_docker_compose_service( + service_section: Dict, docker_compose_dir: Path, - host_environ: Dict=None, - *, use_env_devel=True): - """ Resolves environments in docker-compose's service section, - drops any reference to env_file and sets all - environs 'environment' section + host_environ: Dict = None, + *, + use_env_devel=True +): + """Resolves environments in docker-compose's service section, + drops any reference to env_file and sets all + environs 'environment' section - NOTE: service_section gets modified! + NOTE: service_section gets modified! - SEE https://docs.docker.com/compose/environment-variables/ + SEE https://docs.docker.com/compose/environment-variables/ """ service_environ = {} @@ -77,16 +89,23 @@ def replace_environs_in_docker_compose_service(service_section: Dict, # In VAR=${FOO} matches VAR and FOO # - TODO: add to read defaults envkey = m.groups()[0] - value = host_environ[envkey] # fails when variable in docker-compose is NOT defined + value = host_environ[ + envkey + ] # fails when variable in docker-compose is NOT defined service_environ[key] = value service_section["environment"] = service_environ -def eval_service_environ(docker_compose_path:Path, service_name:str, - host_environ: Dict=None, - image_environ: Dict=None, - *, use_env_devel=True) -> Dict: - """ Deduces a service environment with it runs in a stack from confirmation + +def eval_service_environ( + docker_compose_path: Path, + service_name: str, + host_environ: Dict = None, + image_environ: Dict = None, + *, + use_env_devel=True +) -> Dict: + """Deduces a service environment with it runs in a stack from confirmation :param docker_compose_path: path to stack configuration :type docker_compose_path: Path @@ -104,8 +123,9 @@ def eval_service_environ(docker_compose_path:Path, service_name:str, content = yaml.safe_load(f) service = content["services"][service_name] - replace_environs_in_docker_compose_service(service, docker_compose_dir, - host_environ, use_env_devel=use_env_devel) + replace_environs_in_docker_compose_service( + service, docker_compose_dir, host_environ, use_env_devel=use_env_devel + ) host_environ = host_environ or {} image_environ = image_environ or {} diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index a17fdfb30cef..725843be8595 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -3,22 +3,19 @@ # pylint:disable=redefined-outer-name # pylint:disable=too-many-arguments +import asyncio import json import sys import uuid from pathlib import Path -from typing import Any, Dict, List, Tuple +from typing import Any, Callable, Dict, List, Tuple +import np_helpers import pytest import sqlalchemy as sa -from yarl import URL - -import np_helpers -from simcore_sdk.models.pipeline_models import ( - ComputationalPipeline, - ComputationalTask, -) +from simcore_sdk.models.pipeline_models import ComputationalPipeline, ComputationalTask from simcore_sdk.node_ports import node_config +from yarl import URL current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -47,13 +44,13 @@ def s3_simcore_location() -> str: @pytest.fixture async def filemanager_cfg( - loop, + loop: asyncio.events.AbstractEventLoop, storage_service: URL, devel_environ: Dict, user_id: str, bucket: str, postgres_db, # waits for db and initializes it -): +) -> None: node_config.STORAGE_ENDPOINT = f"{storage_service.host}:{storage_service.port}" node_config.USER_ID = user_id node_config.BUCKET = bucket @@ -71,7 +68,7 @@ def node_uuid() -> str: @pytest.fixture -def file_uuid(project_id, node_uuid) -> str: +def file_uuid(project_id: str, node_uuid: str) -> Callable: def create(file_path: Path, project: str = None, node: str = None): if project is None: project = project_id @@ -100,7 +97,7 @@ def default_configuration( default_configuration_file: Path, project_id, node_uuid, -): +) -> Dict: # prepare database with default configuration json_configuration = default_configuration_file.read_text() @@ -117,16 +114,18 @@ def default_configuration( @pytest.fixture() -def node_link(): - def create_node_link(key: str): +def node_link() -> Callable: + def create_node_link(key: str) -> Dict[str, str]: return {"nodeUuid": "TEST_NODE_UUID", "output": key} yield create_node_link @pytest.fixture() -def store_link(minio_service, bucket, file_uuid, s3_simcore_location): - def create_store_link(file_path: Path, project_id: str = None, node_id: str = None): +def store_link(minio_service, bucket, file_uuid, s3_simcore_location) -> Callable: + def create_store_link( + file_path: Path, project_id: str = None, node_id: str = None + ) -> Dict[str, str]: # upload the file to S3 assert Path(file_path).exists() file_id = file_uuid(file_path, project_id, node_id) @@ -141,19 +140,19 @@ def create_store_link(file_path: Path, project_id: str = None, node_id: str = No @pytest.fixture(scope="function") def special_configuration( - nodeports_config, - bucket, + nodeports_config: None, + bucket: str, postgres_session: sa.orm.session.Session, empty_configuration_file: Path, - project_id, - node_uuid, -): + project_id: str, + node_uuid: str, +) -> Callable: def create_config( inputs: List[Tuple[str, str, Any]] = None, outputs: List[Tuple[str, str, Any]] = None, project_id: str = project_id, node_id: str = node_uuid, - ): + ) -> Tuple[Dict, str, str]: config_dict = json.loads(empty_configuration_file.read_text()) _assign_config(config_dict, "inputs", inputs) _assign_config(config_dict, "outputs", outputs) @@ -174,12 +173,12 @@ def create_config( @pytest.fixture(scope="function") def special_2nodes_configuration( - nodeports_config, - bucket, + nodeports_config: None, + bucket: str, postgres_session: sa.orm.session.Session, empty_configuration_file: Path, - project_id, - node_uuid, + project_id: str, + node_uuid: str, ): def create_config( prev_node_inputs: List[Tuple[str, str, Any]] = None, @@ -189,7 +188,7 @@ def create_config( project_id: str = project_id, previous_node_id: str = node_uuid, node_id: str = "asdasdadsa", - ): + ) -> Tuple[Dict, str, str]: _create_new_pipeline(postgres_session, project_id) # create previous node @@ -225,15 +224,20 @@ def create_config( postgres_session.commit() -def _create_new_pipeline(session, project: str) -> str: +def _create_new_pipeline(postgres_session: sa.orm.session.Session, project: str) -> str: # pylint: disable=no-member new_Pipeline = ComputationalPipeline(project_id=project) - session.add(new_Pipeline) - session.commit() + postgres_session.add(new_Pipeline) + postgres_session.commit() return new_Pipeline.project_id -def _set_configuration(session, project_id: str, node_id: str, json_configuration: str): +def _set_configuration( + postgres_session: sa.orm.session.Session, + project_id: str, + node_id: str, + json_configuration: str, +) -> str: node_uuid = node_id json_configuration = json_configuration.replace("SIMCORE_NODE_UUID", str(node_uuid)) configuration = json.loads(json_configuration) @@ -245,8 +249,8 @@ def _set_configuration(session, project_id: str, node_id: str, json_configuratio inputs=configuration["inputs"], outputs=configuration["outputs"], ) - session.add(new_Node) - session.commit() + postgres_session.add(new_Node) + postgres_session.commit() return node_uuid diff --git a/packages/simcore-sdk/tests/integration/np_helpers.py b/packages/simcore-sdk/tests/integration/np_helpers.py index 947f481765ff..170713ba79a4 100644 --- a/packages/simcore-sdk/tests/integration/np_helpers.py +++ b/packages/simcore-sdk/tests/integration/np_helpers.py @@ -6,39 +6,45 @@ import json import logging from pathlib import Path +from typing import Dict +import sqlalchemy as sa from simcore_sdk.models.pipeline_models import ComputationalTask log = logging.getLogger(__name__) -def update_configuration(session, project_id, node_uuid, new_configuration): - log.debug("Update configuration of pipeline %s, node %s, on session %s", project_id, node_uuid, session) + +def update_configuration( + postgres_session: sa.orm.session.Session, + project_id: str, + node_uuid: str, + new_configuration: Dict, +) -> None: + log.debug( + "Update configuration of pipeline %s, node %s, on session %s", + project_id, + node_uuid, + postgres_session, + ) # pylint: disable=no-member - task = session.query(ComputationalTask).filter( - ComputationalTask.project_id==str(project_id), - ComputationalTask.node_id==str(node_uuid)) - task.update(dict(schema=new_configuration["schema"], inputs=new_configuration["inputs"], outputs=new_configuration["outputs"])) - session.commit() + task = postgres_session.query(ComputationalTask).filter( + ComputationalTask.project_id == str(project_id), + ComputationalTask.node_id == str(node_uuid), + ) + task.update( + dict( + schema=new_configuration["schema"], + inputs=new_configuration["inputs"], + outputs=new_configuration["outputs"], + ) + ) + postgres_session.commit() log.debug("Updated configuration") -def update_config_file(path, config): - - with open(path, "w") as json_file: - json.dump(config, json_file) - - -def get_empty_config(): - return { - "version": "0.1", - "schema": {"inputs":{}, "outputs":{}}, - "inputs": {}, - "outputs": {} - } - - SIMCORE_STORE = "0" -def file_uuid(file_path:Path, project_id:str, node_uuid:str): - file_id = "{}/{}/{}".format(project_id, node_uuid, Path(file_path).name) + +def file_uuid(file_path: Path, project_id: str, node_uuid: str) -> str: + file_id = f"{project_id}/{node_uuid}/{Path(file_path).name}" return file_id diff --git a/packages/simcore-sdk/tests/integration/test_dbmanager.py b/packages/simcore-sdk/tests/integration/test_dbmanager.py index e3846c10678d..a5a5fe370cc2 100644 --- a/packages/simcore-sdk/tests/integration/test_dbmanager.py +++ b/packages/simcore-sdk/tests/integration/test_dbmanager.py @@ -2,9 +2,10 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import asyncio import json from pathlib import Path -from typing import Dict +from typing import Callable, Dict from simcore_sdk.node_ports import config from simcore_sdk.node_ports.dbmanager import DBManager @@ -17,7 +18,9 @@ async def test_db_manager_read_config( - loop, nodeports_config, default_configuration: Dict + loop: asyncio.events.AbstractEventLoop, + nodeports_config: None, + default_configuration: Dict, ): db_manager = DBManager() ports_configuration_str = await db_manager.get_ports_configuration_from_node_uuid( @@ -29,7 +32,10 @@ async def test_db_manager_read_config( async def test_db_manager_write_config( - loop, nodeports_config, special_configuration, default_configuration_file: Path + loop: asyncio.events.AbstractEventLoop, + nodeports_config: None, + special_configuration: Callable, + default_configuration_file: Path, ): # create an empty config special_configuration() diff --git a/packages/simcore-sdk/tests/integration/test_filemanager.py b/packages/simcore-sdk/tests/integration/test_filemanager.py index b4b990d54408..3955f130f8aa 100644 --- a/packages/simcore-sdk/tests/integration/test_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_filemanager.py @@ -15,7 +15,12 @@ async def test_valid_upload_download( - tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location + tmpdir: Path, + bucket: str, + filemanager_cfg: None, + user_id: str, + file_uuid: str, + s3_simcore_location: str, ): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") @@ -37,7 +42,12 @@ async def test_valid_upload_download( async def test_invalid_file_path( - tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location + tmpdir: Path, + bucket: str, + filemanager_cfg: None, + user_id: str, + file_uuid: str, + s3_simcore_location: str, ): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") @@ -60,7 +70,11 @@ async def test_invalid_file_path( async def test_invalid_fileid( - tmpdir, bucket, filemanager_cfg, user_id, s3_simcore_location + tmpdir: Path, + bucket: str, + filemanager_cfg: None, + user_id: str, + s3_simcore_location: str, ): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") @@ -88,7 +102,12 @@ async def test_invalid_fileid( async def test_invalid_store( - tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location + tmpdir: Path, + bucket: str, + filemanager_cfg: None, + user_id: str, + file_uuid: str, + s3_simcore_location: str, ): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") diff --git a/packages/simcore-sdk/tests/integration/test_nodeports.py b/packages/simcore-sdk/tests/integration/test_nodeports.py index b18e4f1b2c9b..9dc74ca41e73 100644 --- a/packages/simcore-sdk/tests/integration/test_nodeports.py +++ b/packages/simcore-sdk/tests/integration/test_nodeports.py @@ -7,12 +7,15 @@ import filecmp import tempfile from pathlib import Path +from typing import Callable, Dict, Type +import np_helpers # pylint: disable=no-name-in-module import pytest +import sqlalchemy as sa from simcore_sdk import node_ports from simcore_sdk.node_ports import exceptions - -import np_helpers # pylint: disable=no-name-in-module +from simcore_sdk.node_ports._item import ItemConcreteValue +from simcore_sdk.node_ports.nodeports import Nodeports core_services = ["postgres", "storage"] @@ -20,7 +23,7 @@ async def _check_port_valid( - ports, config_dict: dict, port_type: str, key_name: str, key + ports: Nodeports, config_dict: Dict, port_type: str, key_name: str, key: str ): assert (await getattr(ports, port_type))[key].key == key_name # check required values @@ -68,7 +71,7 @@ async def _check_port_valid( assert (await getattr(ports, port_type))[key].value == None -async def _check_ports_valid(ports, config_dict: dict, port_type: str): +async def _check_ports_valid(ports: Nodeports, config_dict: Dict, port_type: str): for key in config_dict["schema"][port_type].keys(): # test using "key" name await _check_port_valid(ports, config_dict, port_type, key, key) @@ -77,19 +80,19 @@ async def _check_ports_valid(ports, config_dict: dict, port_type: str): await _check_port_valid(ports, config_dict, port_type, key, key_index) -async def check_config_valid(ports, config_dict: dict): +async def check_config_valid(ports: Nodeports, config_dict: Dict): await _check_ports_valid(ports, config_dict, "inputs") await _check_ports_valid(ports, config_dict, "outputs") async def test_default_configuration( - loop, default_configuration + default_configuration: Dict, ): # pylint: disable=W0613, W0621 config_dict = default_configuration await check_config_valid(await node_ports.ports(), config_dict) -async def test_invalid_ports(loop, special_configuration): +async def test_invalid_ports(special_configuration: Callable): config_dict, _, _ = special_configuration() PORTS = await node_ports.ports() await check_config_valid(PORTS, config_dict) @@ -120,7 +123,10 @@ async def test_invalid_ports(loop, special_configuration): ], ) async def test_port_value_accessors( - special_configuration, item_type, item_value, item_pytype + special_configuration: Callable, + item_type: str, + item_value: ItemConcreteValue, + item_pytype: Type, ): # pylint: disable=W0613, W0621 item_key = "some key" config_dict, _, _ = special_configuration( @@ -152,14 +158,14 @@ async def test_port_value_accessors( ], ) async def test_port_file_accessors( - special_configuration, - filemanager_cfg, - s3_simcore_location, - bucket, - item_type, - item_value, - item_pytype, - config_value, + special_configuration: Callable, + filemanager_cfg: None, + s3_simcore_location: str, + bucket: str, + item_type: str, + item_value: str, + item_pytype: Type, + config_value: Dict[str, str], ): # pylint: disable=W0613, W0621 config_dict, project_id, node_uuid = special_configuration( inputs=[("in_1", item_type, config_value)], @@ -188,7 +194,10 @@ async def test_port_file_accessors( assert filecmp.cmp(item_value, await (await PORTS.outputs)["out_34"].get()) -async def test_adding_new_ports(special_configuration, postgres_session): +async def test_adding_new_ports( + special_configuration: Callable, + postgres_session: sa.orm.session.Session, +): config_dict, project_id, node_uuid = special_configuration() PORTS = await node_ports.ports() await check_config_valid(PORTS, config_dict) @@ -230,7 +239,10 @@ async def test_adding_new_ports(special_configuration, postgres_session): await check_config_valid(PORTS, config_dict) -async def test_removing_ports(special_configuration, postgres_session): +async def test_removing_ports( + special_configuration: Callable, + postgres_session: sa.orm.session.Session, +): config_dict, project_id, node_uuid = special_configuration( inputs=[("in_14", "integer", 15), ("in_17", "boolean", False)], outputs=[("out_123", "string", "blahblah"), ("out_2", "number", -12.3)], @@ -269,7 +281,11 @@ async def test_removing_ports(special_configuration, postgres_session): ], ) async def test_get_value_from_previous_node( - special_2nodes_configuration, node_link, item_type, item_value, item_pytype + special_2nodes_configuration: Callable, + node_link: Callable, + item_type: str, + item_value: ItemConcreteValue, + item_pytype: Type, ): config_dict, _, _ = special_2nodes_configuration( prev_node_outputs=[("output_123", item_type, item_value)], @@ -292,15 +308,15 @@ async def test_get_value_from_previous_node( ], ) async def test_get_file_from_previous_node( - special_2nodes_configuration, - project_id, - node_uuid, - filemanager_cfg, - node_link, - store_link, - item_type, - item_value, - item_pytype, + special_2nodes_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + node_link: Callable, + store_link: Callable, + item_type: str, + item_value: str, + item_pytype: Type, ): config_dict, _, _ = special_2nodes_configuration( prev_node_outputs=[ @@ -332,17 +348,17 @@ async def test_get_file_from_previous_node( ], ) async def test_get_file_from_previous_node_with_mapping_of_same_key_name( - special_2nodes_configuration, - project_id, - node_uuid, - filemanager_cfg, - node_link, - store_link, - postgres_session, - item_type, - item_value, - item_alias, - item_pytype, + special_2nodes_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + node_link: Callable, + store_link: Callable, + postgres_session: sa.orm.session.Session, + item_type: str, + item_value: str, + item_alias: str, + item_pytype: Type, ): config_dict, _, this_node_uuid = special_2nodes_configuration( prev_node_outputs=[ @@ -378,18 +394,18 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name( ], ) async def test_file_mapping( - special_configuration, - project_id, - node_uuid, - filemanager_cfg, - s3_simcore_location, - bucket, - store_link, - postgres_session, - item_type, - item_value, - item_alias, - item_pytype, + special_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + s3_simcore_location: str, + bucket: str, + store_link: Callable, + postgres_session: sa.orm.session.Session, + item_type: str, + item_value: str, + item_alias: str, + item_pytype: Type, ): config_dict, project_id, node_uuid = special_configuration( inputs=[("in_1", item_type, store_link(item_value, project_id, node_uuid))], diff --git a/packages/simcore-sdk/tests/unit/test_data_item.py b/packages/simcore-sdk/tests/unit/test_data_item.py index e4b382d61334..a17e5f35d70c 100644 --- a/packages/simcore-sdk/tests/unit/test_data_item.py +++ b/packages/simcore-sdk/tests/unit/test_data_item.py @@ -4,7 +4,7 @@ import pytest from simcore_sdk.node_ports import exceptions -from simcore_sdk.node_ports._data_item import DataItem +from simcore_sdk.node_ports._data_item import DataItem, DataItemValue @pytest.mark.parametrize( @@ -37,7 +37,7 @@ (None), ], ) -def test_default_item(item_value): +def test_default_item(item_value: DataItemValue): with pytest.raises(exceptions.InvalidProtocolError): DataItem() with pytest.raises(exceptions.InvalidProtocolError): diff --git a/packages/simcore-sdk/tests/unit/test_data_manager.py b/packages/simcore-sdk/tests/unit/test_data_manager.py index 19a91db929e7..d69d3d3fbc64 100644 --- a/packages/simcore-sdk/tests/unit/test_data_manager.py +++ b/packages/simcore-sdk/tests/unit/test_data_manager.py @@ -2,20 +2,22 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import asyncio from asyncio import Future from filecmp import cmpfiles from pathlib import Path -from shutil import make_archive, unpack_archive, copy +from shutil import copy, make_archive, unpack_archive +from typing import Callable, List import pytest from simcore_sdk.node_data import data_manager @pytest.fixture -def create_files(): +def create_files() -> Callable: created_files = [] - def _create_files(number: int, folder: Path): + def _create_files(number: int, folder: Path) -> List[Path]: for i in range(number): file_path = folder / "{}.test".format(i) @@ -30,7 +32,9 @@ def _create_files(number: int, folder: Path): file_path.unlink() -async def test_push_folder(loop, mocker, tmpdir, create_files): +async def test_push_folder( + loop: asyncio.events.AbstractEventLoop, mocker, tmpdir: Path, create_files: Callable +): # create some files assert tmpdir.exists() @@ -93,7 +97,9 @@ async def test_push_folder(loop, mocker, tmpdir, create_files): assert not errors -async def test_push_file(loop, mocker, tmpdir, create_files): +async def test_push_file( + loop: asyncio.events.AbstractEventLoop, mocker, tmpdir: Path, create_files: Callable +): mock_filemanager = mocker.patch( "simcore_sdk.node_data.data_manager.filemanager", spec=True ) @@ -122,7 +128,9 @@ async def test_push_file(loop, mocker, tmpdir, create_files): mock_filemanager.reset_mock() -async def test_pull_folder(loop, mocker, tmpdir, create_files): +async def test_pull_folder( + loop: asyncio.events.AbstractEventLoop, mocker, tmpdir: Path, create_files: Callable +): assert tmpdir.exists() # create a folder to compress from test_control_folder = Path(tmpdir) / "test_control_folder" @@ -186,7 +194,9 @@ async def test_pull_folder(loop, mocker, tmpdir, create_files): assert not errors -async def test_pull_file(loop, mocker, tmpdir, create_files): +async def test_pull_file( + loop: asyncio.events.AbstractEventLoop, mocker, tmpdir: Path, create_files: Callable +): file_path = create_files(1, Path(tmpdir))[0] assert file_path.exists() assert file_path.is_file() diff --git a/packages/simcore-sdk/tests/unit/test_item.py b/packages/simcore-sdk/tests/unit/test_item.py index 0f12a234e044..3d802c9efec8 100644 --- a/packages/simcore-sdk/tests/unit/test_item.py +++ b/packages/simcore-sdk/tests/unit/test_item.py @@ -4,12 +4,12 @@ # pylint:disable=no-member from pathlib import Path -from typing import Callable, Dict, Union +from typing import Callable import pytest from simcore_sdk.node_ports import config, data_items_utils, exceptions, filemanager from simcore_sdk.node_ports._data_item import DataItem -from simcore_sdk.node_ports._item import Item +from simcore_sdk.node_ports._item import DataItemValue, Item, ItemConcreteValue from simcore_sdk.node_ports._schema_item import SchemaItem from utils_futures import future_with_result @@ -19,7 +19,7 @@ def node_ports_config(): config.STORAGE_ENDPOINT = "storage:8080" -def create_item(item_type: str, item_value): +def create_item(item_type: str, item_value: DataItemValue) -> Item: key = "some key" return Item( SchemaItem( @@ -78,7 +78,7 @@ async def test_valid_type_empty_value(item_type: str): @pytest.fixture async def file_link_mock( - monkeypatch, item_type: str, item_value: Union[int, float, bool, str, Dict] + monkeypatch, item_type: str, item_value: DataItemValue ) -> Callable: async def fake_download_file(*args, **kwargs) -> Path: return_value = "mydefault" @@ -128,7 +128,7 @@ async def test_valid_type( loop, file_link_mock: Callable, item_type: str, - item_value: Union[int, float, bool, str, Dict], + item_value: DataItemValue, ): item = create_item(item_type, item_value) if not data_items_utils.is_value_link(item_value): @@ -203,7 +203,7 @@ async def test_valid_type( ), ], ) -async def test_invalid_type(item_type, item_value): +async def test_invalid_type(item_type: str, item_value: DataItemValue): # pylint: disable=W0612 with pytest.raises( exceptions.InvalidItemTypeError, match=rf"Invalid item type, .*[{item_type}]" @@ -216,13 +216,15 @@ async def test_invalid_type(item_type, item_value): [ ("integer", 26, 26), ("number", -746.4748, -746.4748), - # ("data:*/*", __file__, {"store":"s3-z43", "path":"undefined/undefined/{filename}".format(filename=Path(__file__).name)}), ("boolean", False, False), ("string", "test-string", "test-string"), ], ) async def test_set_new_value( - item_type, item_value_to_set, expected_value, mocker + item_type: str, + item_value_to_set: ItemConcreteValue, + expected_value: ItemConcreteValue, + mocker, ): # pylint: disable=W0613 mock_method = mocker.Mock(return_value=future_with_result("")) item = create_item(item_type, None) @@ -237,13 +239,14 @@ async def test_set_new_value( [ ("integer", -746.4748), ("number", "a string"), + ("data:*/*", Path(__file__).parent), ("data:*/*", str(Path(__file__).parent)), ("boolean", 123), ("string", True), ], ) async def test_set_new_invalid_value( - item_type, item_value_to_set + item_type: str, item_value_to_set: ItemConcreteValue ): # pylint: disable=W0613 item = create_item(item_type, None) assert await item.get() is None diff --git a/packages/simcore-sdk/tests/unit/test_itemstlist.py b/packages/simcore-sdk/tests/unit/test_itemstlist.py index b1244adeb62a..7c4723c1b4b6 100644 --- a/packages/simcore-sdk/tests/unit/test_itemstlist.py +++ b/packages/simcore-sdk/tests/unit/test_itemstlist.py @@ -3,8 +3,9 @@ # pylint:disable=redefined-outer-name # pylint:disable=no-member -import pytest +from typing import Dict, Union +import pytest from simcore_sdk.node_ports._data_item import DataItem from simcore_sdk.node_ports._data_items_list import DataItemsList from simcore_sdk.node_ports._item import Item @@ -14,7 +15,9 @@ from utils_futures import future_with_result -def create_item(key, item_type, item_value): +def create_item( + key: str, item_type: str, item_value: Union[int, float, bool, str, Dict] +): return Item( SchemaItem( key=key, From a854b5c30ba030a3673b18882f2b2715a0e7b7b5 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 30 Nov 2020 21:02:42 +0100 Subject: [PATCH 07/74] repair merge --- packages/simcore-sdk/requirements/_test.in | 2 -- packages/simcore-sdk/requirements/_test.txt | 13 ++----------- packages/simcore-sdk/requirements/_tools.txt | 10 +++++----- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index 76d6c3325a82..2abd0641c010 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -8,11 +8,9 @@ -c _base.txt # testing -alembic coverage pytest pytest-aiohttp # incompatible with pytest-asyncio. See https://github.com/pytest-dev/pytest-asyncio/issues/76 -pytest-black pytest-cov pytest-icdiff pytest-instafail diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index adf1c4a1c2ec..2011027397e1 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -6,15 +6,12 @@ # aiohttp==3.7.3 # via -c requirements/_base.txt, aioresponses, pytest-aiohttp aioresponses==0.7.1 # via -r requirements/_test.in -alembic==1.4.3 # via -r requirements/_test.in apipkg==1.5 # via execnet -appdirs==1.4.4 # via black astroid==2.4.2 # via pylint async-timeout==3.0.1 # via -c requirements/_base.txt, aiohttp attrs==20.3.0 # via -c requirements/_base.txt, aiohttp, pytest certifi==2020.12.5 # via -c requirements/_base.txt, requests chardet==3.0.4 # via -c requirements/_base.txt, aiohttp, requests -click==7.1.2 # via black coverage==5.3 # via -r requirements/_test.in, coveralls, pytest-cov coveralls==2.2.0 # via -r requirements/_test.in docker==4.4.0 # via -r requirements/_test.in @@ -32,12 +29,10 @@ multidict==5.1.0 # via -c requirements/_base.txt, aiohttp, yarl packaging==20.7 # via pytest, pytest-sugar pluggy==0.13.1 # via pytest pprintpp==0.4.0 # via pytest-icdiff -psycopg2-binary==2.8.6 # via -c requirements/_base.txt, sqlalchemy py==1.9.0 # via pytest, pytest-forked pylint==2.6.0 # via -r requirements/_test.in pyparsing==2.4.7 # via packaging pytest-aiohttp==0.3.0 # via -r requirements/_test.in -pytest-black==0.3.12 # via -r requirements/_test.in pytest-cov==2.10.1 # via -r requirements/_test.in pytest-forked==1.3.0 # via pytest-xdist pytest-icdiff==0.5 # via -r requirements/_test.in @@ -46,14 +41,10 @@ pytest-mock==3.3.1 # via -r requirements/_test.in pytest-runner==5.2 # via -r requirements/_test.in pytest-sugar==0.9.4 # via -r requirements/_test.in pytest-xdist==2.1.0 # via -r requirements/_test.in -pytest==6.1.2 # via -r requirements/_test.in, pytest-aiohttp, pytest-black, pytest-cov, pytest-forked, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar, pytest-xdist -python-dateutil==2.8.1 # via alembic +pytest==6.1.2 # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-forked, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar, pytest-xdist python-dotenv==0.15.0 # via -r requirements/_test.in -python-editor==1.0.4 # via alembic -regex==2020.11.13 # via black requests==2.25.0 # via -r requirements/_test.in, coveralls, docker -six==1.15.0 # via -c requirements/_base.txt, astroid, docker, packaging, python-dateutil, websocket-client -sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via -c requirements/_base.txt, alembic +six==1.15.0 # via -c requirements/_base.txt, astroid, docker, packaging, websocket-client termcolor==1.1.0 # via pytest-sugar toml==0.10.2 # via pylint, pytest typed-ast==1.4.1 # via astroid diff --git a/packages/simcore-sdk/requirements/_tools.txt b/packages/simcore-sdk/requirements/_tools.txt index ea5a8ac2953a..17b080dc9f0f 100644 --- a/packages/simcore-sdk/requirements/_tools.txt +++ b/packages/simcore-sdk/requirements/_tools.txt @@ -4,8 +4,8 @@ # # pip-compile --output-file=requirements/_tools.txt requirements/_tools.in # -appdirs==1.4.4 # via -c requirements/_test.txt, black, virtualenv -black==20.8b1 # via -c requirements/_test.txt, -r requirements/../../../requirements/devenv.txt +appdirs==1.4.4 # via black, virtualenv +black==20.8b1 # via -r requirements/../../../requirements/devenv.txt bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt cfgv==3.2.0 # via pre-commit click==7.1.2 # via black, pip-tools @@ -16,13 +16,13 @@ identify==1.5.10 # via pre-commit importlib-metadata==3.1.1 # via -c requirements/_base.txt, -c requirements/_test.txt, pre-commit, virtualenv importlib-resources==3.3.0 # via pre-commit, virtualenv isort==5.6.4 # via -c requirements/_test.txt, -r requirements/../../../requirements/devenv.txt -mypy-extensions==0.4.3 # via -c requirements/_test.txt, black +mypy-extensions==0.4.3 # via black nodeenv==1.5.0 # via pre-commit -pathspec==0.8.1 # via -c requirements/_test.txt, black +pathspec==0.8.1 # via black pip-tools==5.4.0 # via -r requirements/../../../requirements/devenv.txt pre-commit==2.9.3 # via -r requirements/../../../requirements/devenv.txt pyyaml==5.3.1 # via -c requirements/_base.txt, pre-commit -regex==2020.11.13 # via -c requirements/_test.txt, black +regex==2020.11.13 # via black six==1.15.0 # via -c requirements/_base.txt, -c requirements/_test.txt, pip-tools, virtualenv toml==0.10.2 # via -c requirements/_test.txt, black, pre-commit typed-ast==1.4.1 # via -c requirements/_test.txt, black From 6299a11fd7ff052f5d9cd47b49a61e4ebd2c8886 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 30 Nov 2020 21:06:52 +0100 Subject: [PATCH 08/74] missing alembic for testing --- packages/simcore-sdk/requirements/_test.in | 1 + packages/simcore-sdk/requirements/_test.txt | 7 ++++++- packages/simcore-sdk/tests/integration/test_dbmanager.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index 2abd0641c010..3f06bfbcc258 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -20,6 +20,7 @@ pytest-sugar pytest-xdist # mockups/fixtures +alembic aioresponses requests docker diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index 2011027397e1..d2925688019d 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -6,6 +6,7 @@ # aiohttp==3.7.3 # via -c requirements/_base.txt, aioresponses, pytest-aiohttp aioresponses==0.7.1 # via -r requirements/_test.in +alembic==1.4.3 # via -r requirements/_test.in apipkg==1.5 # via execnet astroid==2.4.2 # via pylint async-timeout==3.0.1 # via -c requirements/_base.txt, aiohttp @@ -29,6 +30,7 @@ multidict==5.1.0 # via -c requirements/_base.txt, aiohttp, yarl packaging==20.7 # via pytest, pytest-sugar pluggy==0.13.1 # via pytest pprintpp==0.4.0 # via pytest-icdiff +psycopg2-binary==2.8.6 # via -c requirements/_base.txt, sqlalchemy py==1.9.0 # via pytest, pytest-forked pylint==2.6.0 # via -r requirements/_test.in pyparsing==2.4.7 # via packaging @@ -42,9 +44,12 @@ pytest-runner==5.2 # via -r requirements/_test.in pytest-sugar==0.9.4 # via -r requirements/_test.in pytest-xdist==2.1.0 # via -r requirements/_test.in pytest==6.1.2 # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-forked, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar, pytest-xdist +python-dateutil==2.8.1 # via alembic python-dotenv==0.15.0 # via -r requirements/_test.in +python-editor==1.0.4 # via alembic requests==2.25.0 # via -r requirements/_test.in, coveralls, docker -six==1.15.0 # via -c requirements/_base.txt, astroid, docker, packaging, websocket-client +six==1.15.0 # via -c requirements/_base.txt, astroid, docker, packaging, python-dateutil, websocket-client +sqlalchemy[postgresql_psycopg2binary]==1.3.20 # via -c requirements/_base.txt, alembic termcolor==1.1.0 # via pytest-sugar toml==0.10.2 # via pylint, pytest typed-ast==1.4.1 # via astroid diff --git a/packages/simcore-sdk/tests/integration/test_dbmanager.py b/packages/simcore-sdk/tests/integration/test_dbmanager.py index a5a5fe370cc2..e2b50beacab7 100644 --- a/packages/simcore-sdk/tests/integration/test_dbmanager.py +++ b/packages/simcore-sdk/tests/integration/test_dbmanager.py @@ -14,7 +14,7 @@ "postgres", ] -ops_services = ["minio"] +ops_services = [] async def test_db_manager_read_config( From 32f739c50c88dbd15f9e335479f8a430c98a53b5 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 30 Nov 2020 23:32:30 +0100 Subject: [PATCH 09/74] set back minio --- packages/simcore-sdk/tests/integration/test_dbmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/simcore-sdk/tests/integration/test_dbmanager.py b/packages/simcore-sdk/tests/integration/test_dbmanager.py index e2b50beacab7..a5a5fe370cc2 100644 --- a/packages/simcore-sdk/tests/integration/test_dbmanager.py +++ b/packages/simcore-sdk/tests/integration/test_dbmanager.py @@ -14,7 +14,7 @@ "postgres", ] -ops_services = [] +ops_services = ["minio"] async def test_db_manager_read_config( From cfc2e981bccfa02807c7b768de7fe796befbdebc Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 10:08:16 +0100 Subject: [PATCH 10/74] Squashed commit of the following: commit c850a758baa3eaca0af7f879f8f817509ba83bab Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu Dec 3 10:07:21 2020 +0100 fixing imports commit c8873d2cc49f7369d2e446a859417b3855dc527b Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu Dec 3 09:55:29 2020 +0100 integration tests for nodeports2 commit 737496b84fb188b05ea0d7711279129bc250c131 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu Dec 3 09:54:02 2020 +0100 fix imports commit 468eabaf0fd6b5fb59a38fc642871e1f064a3504 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu Dec 3 09:49:28 2020 +0100 restore old node ports v1 commit 1c2789eff604000475d929575fab3adffad46798 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu Dec 3 09:46:32 2020 +0100 nodeports v2 commit 52df6eddbe157646fa9ab7c7b9b3d7666c7dcd1d Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu Dec 3 09:31:17 2020 +0100 all integration tests passing commit f6ab328b50a489662045442f74677e75a22affc3 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu Dec 3 09:20:17 2020 +0100 auto-update from/to database commit 9951f4ebfa229166d9caabba32eb7f0a58fbd9a9 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu Dec 3 08:46:43 2020 +0100 missing return value commit aecb512a32eb292280f4644c709a1db8f6f73b59 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu Dec 3 00:31:59 2020 +0100 tests almost passing commit c46d156c1d91bb12383233cda03143b8f25f4559 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed Dec 2 23:32:29 2020 +0100 more fixes commit c6ef1f50871f79459e27aed62ac7ef1b6337cd8e Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed Dec 2 23:19:47 2020 +0100 direct set works commit 5b96c07d73624d43bd70f47486aa516c153e0db7 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed Dec 2 21:54:34 2020 +0100 replacing old parts commit 5e226e409731178019dfb30103a55d8e0f8fc452 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed Dec 2 15:38:11 2020 +0100 refactor commit b874ccb629c612d7b61d680d7d8bc2c0630e75ec Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed Dec 2 00:01:13 2020 +0100 use pydantic in schema item list commit 499c5ccbc32b296bbf51faf8d4e30ca1acbe9796 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue Dec 1 23:51:56 2020 +0100 data_item_list now uses pydantic commit f318dfd0ee5758551aaee66d1ad8b3bc38522aa7 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue Dec 1 22:37:14 2020 +0100 fixed annotation commit dfb6bd2a28e93f228087162c76fc45a71469b494 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue Dec 1 22:22:40 2020 +0100 use pydantic in serialization commit 882b9b3c6c734e537dcede3f1aa65c105976e025 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue Dec 1 22:06:11 2020 +0100 some cleaning commit 6a03854c37ca36c2bdc24ce6b743ba6caef980bd Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue Dec 1 21:59:40 2020 +0100 use pydantic in _item commit 98ba241ecb72bf7566d38df926b10048a2735de9 Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue Dec 1 00:21:10 2020 +0100 replaced named typle by pydantic commit 63bb20cbbbc076966ff26f047ca323fc577f060c Author: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue Dec 1 00:12:19 2020 +0100 use pydantic in data_item --- .../src/simcore_sdk/node_ports_v2/__init__.py | 26 + .../src/simcore_sdk/node_ports_v2/links.py | 36 ++ .../simcore_sdk/node_ports_v2/nodeports_v2.py | 89 ++++ .../src/simcore_sdk/node_ports_v2/port.py | 143 ++++++ .../simcore_sdk/node_ports_v2/port_utils.py | 99 ++++ .../node_ports_v2/serialization_v2.py | 76 +++ .../tests/integration/test_nodeports2.py | 448 ++++++++++++++++++ 7 files changed, 917 insertions(+) create mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py create mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py create mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py create mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py create mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py create mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py create mode 100644 packages/simcore-sdk/tests/integration/test_nodeports2.py diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py new file mode 100644 index 000000000000..4ff38a59920c --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py @@ -0,0 +1,26 @@ +import logging +from typing import Optional + +from ..node_ports import config as node_config +from ..node_ports import exceptions +from ..node_ports.dbmanager import DBManager +from .nodeports_v2 import Nodeports +from .serialization_v2 import create_nodeports_from_db + +# nodeports is a library for accessing data linked to the node +# in that sense it should not log stuff unless the application code wants it to be so. +log = logging.getLogger(__name__) +log.addHandler(logging.NullHandler()) + + +async def ports(db_manager: Optional[DBManager] = None) -> Nodeports: + # FIXME: warning every dbmanager create a new db engine! + if db_manager is None: # NOTE: keeps backwards compatibility + db_manager = DBManager() + + return await create_nodeports_from_db( + db_manager, node_config.NODE_UUID, auto_update=True + ) + + +__all__ = ["ports", "node_config", "exceptions"] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py new file mode 100644 index 000000000000..f6c5e37c2f68 --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py @@ -0,0 +1,36 @@ +from pathlib import Path +from typing import Optional, Union + +from pydantic import ( + AnyUrl, + BaseModel, + Field, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, +) + + +class PortLink(BaseModel): + node_uuid: str = Field(..., alias="nodeUuid") + output: str + + +class DownloadLink(BaseModel): + download_link: AnyUrl = Field(..., alias="downloadLink") + label: Optional[str] + + +class FileLink(BaseModel): + store: Union[str, int] + path: str + dataset: Optional[str] + label: Optional[str] + + +DataItemValue = Union[ + StrictBool, StrictInt, StrictFloat, StrictStr, DownloadLink, PortLink, FileLink +] + +ItemConcreteValue = Union[int, float, bool, str, Path] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py new file mode 100644 index 000000000000..3c6fbba3c906 --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -0,0 +1,89 @@ +import logging +from pathlib import Path +from typing import Any, Callable, Coroutine, Type + +from pydantic import BaseModel, Field + +from ..node_ports.dbmanager import DBManager +from ..node_ports.exceptions import PortNotFound, UnboundPortError +from .links import ItemConcreteValue +from .port import InputsList, OutputsList +from .port_utils import is_file_type + +log = logging.getLogger(__name__) + + +class Nodeports(BaseModel): + internal_inputs: InputsList = Field(..., alias="inputs") + internal_outputs: OutputsList = Field(..., alias="outputs") + db_manager: DBManager + node_uuid: str + save_to_db_cb: Callable[["Nodeports"], Coroutine[Any, Any, None]] + node_port_creator_cb: Callable[[DBManager, str], Coroutine[Any, Any, "Nodeports"]] + auto_update: bool = False + + class Config: + arbitrary_types_allowed = True + + def __init__(self, **data): + super().__init__(**data) + # let's pass ourselves down + for input_key in self.internal_inputs: + self.internal_inputs[input_key]._node_ports = self + for output_key in self.internal_outputs: + self.internal_outputs[output_key]._node_ports = self + + @property + async def inputs(self) -> InputsList: + log.debug("Getting inputs with autoupdate: %s", self.auto_update) + if self.auto_update: + await self._auto_update_from_db() + return self.internal_inputs + + @property + async def outputs(self) -> OutputsList: + log.debug("Getting outputs with autoupdate: %s", self.auto_update) + if self.auto_update: + await self._auto_update_from_db() + return self.internal_outputs + + async def get(self, item_key: str) -> ItemConcreteValue: + try: + return await (await self.inputs)[item_key].get() + except UnboundPortError: + # not available try outputs + pass + # if this fails it will raise an exception + return await (await self.outputs)[item_key].get() + + async def set(self, item_key: str, item_value): + try: + await (await self.inputs)[item_key].set(item_value) + except UnboundPortError: + # not available try outputs + pass + # if this fails it will raise an exception + return await (await self.outputs)[item_key].set(item_value) + + async def set_file_by_keymap(self, item_value: Path): + for output in (await self.outputs).values(): + if is_file_type(output.type) and output.fileToKeyMap: + if item_value.name in output.fileToKeyMap: + await output.set(item_value) + return + raise PortNotFound(msg=f"output port for item {item_value} not found") + + async def _node_ports_creator_cb(self, node_uuid: str) -> Type["NodePorts"]: + return await self.node_port_creator_cb(self.db_manager, node_uuid) + + async def _auto_update_from_db(self) -> None: + # get the newest from the DB + updated_node_ports = await self._node_ports_creator_cb(self.node_uuid) + # update our stuff + self.internal_inputs = updated_node_ports.internal_inputs + self.internal_outputs = updated_node_ports.internal_outputs + # let's pass ourselves down + for input_key in self.internal_inputs: + self.internal_inputs[input_key]._node_ports = self + for output_key in self.internal_outputs: + self.internal_outputs[output_key]._node_ports = self diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py new file mode 100644 index 000000000000..a9015eb0d72f --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -0,0 +1,143 @@ +import logging +from pathlib import Path +from pprint import pformat +from typing import Dict, Optional, Tuple, Type, Union + +from pydantic import ( + BaseModel, + PrivateAttr, + StrictBool, + StrictFloat, + StrictInt, + validator, +) + +from ..node_ports.exceptions import InvalidItemTypeError, UnboundPortError +from . import port_utils +from .links import DataItemValue, DownloadLink, FileLink, ItemConcreteValue, PortLink + +log = logging.getLogger(__name__) + + +TYPE_TO_PYTYPE: Dict[str, Type[ItemConcreteValue]] = { + "integer": int, + "number": float, + "boolean": bool, + "string": str, +} + + +class Port(BaseModel): + key: str + label: str + description: str + type: str + displayOrder: float + fileToKeyMap: Optional[Dict[str, str]] = None + defaultValue: Optional[Union[StrictBool, StrictInt, StrictFloat, str]] = None + widget: Optional[Dict] = None + + value: Optional[DataItemValue] + + _py_value_type: Tuple[Type[ItemConcreteValue]] = PrivateAttr() + _py_value_converter: Type[ItemConcreteValue] = PrivateAttr() + _node_ports = PrivateAttr() + + @validator("value", always=True) + @classmethod + def ensure_value(cls, v, values): + if not v and values.get("defaultValue"): + return values["defaultValue"] + return v + + def __init__(self, **data): + super().__init__(**data) + # let's define the converter + self._py_value_type = ( + (Path, str) + if port_utils.is_file_type(self.type) + else (TYPE_TO_PYTYPE[self.type]) + ) + self._py_value_converter = ( + Path if port_utils.is_file_type(self.type) else TYPE_TO_PYTYPE[self.type] + ) + + async def get(self) -> ItemConcreteValue: + log.debug( + "getting %s[%s] with value %s", self.key, self.type, pformat(self.value) + ) + + if self.value is None: + return None + + value = None + if isinstance(self.value, PortLink): + # this is a link to another node + value = await port_utils.get_value_from_link( + self.key, + self.value, + self.fileToKeyMap, + self._node_ports._node_ports_creator_cb, + ) + elif isinstance(self.value, FileLink): + # this is a link from storage + value = await port_utils.pull_file_from_store( + self.key, self.fileToKeyMap, self.value + ) + elif isinstance(self.value, DownloadLink): + # this is a downloadable link + value = await port_utils.pull_file_from_download_link( + self.key, self.fileToKeyMap, self.value + ) + else: + # this is directly the value + value = self.value + + return self._py_value_converter(value) + + async def set(self, new_value: ItemConcreteValue): + log.debug("setting %s[%s] with value %s", self.key, self.type, new_value) + if not isinstance(new_value, self._py_value_type): + raise InvalidItemTypeError(self.type, new_value) + + # convert the concrete value to a data value + data_value = self._py_value_converter(new_value) + if port_utils.is_file_type(self.type): + if not data_value.exists() or not data_value.is_file(): + raise InvalidItemTypeError(self.type, new_value) + data_value: FileLink = await port_utils.push_file_to_store(data_value) + + self.value = data_value + await self._node_ports.save_to_db_cb(self._node_ports) + + +PortKey = str + + +class PortsMapping(BaseModel): + __root__: Dict[PortKey, Port] + + def __getitem__(self, key: Union[int, PortKey]) -> Port: + if isinstance(key, int): + if key < len(self.__root__): + key = list(self.__root__.keys())[key] + if not key in self.__root__: + raise UnboundPortError(key) + return self.__root__[key] + + def __iter__(self): + return iter(self.__root__) + + def items(self): + return self.__root__.items() + + def values(self): + return self.__root__.values() + + +class InputsList(PortsMapping): + __root__: Dict[PortKey, Port] + + +class OutputsList(PortsMapping): + __root__: Dict[PortKey, Port] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py new file mode 100644 index 000000000000..8cbefa73f54a --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -0,0 +1,99 @@ +import logging +import shutil +from pathlib import Path +from typing import Any, Callable, Coroutine, Dict + +from yarl import URL + +from ..node_ports import config, data_items_utils, filemanager +from .links import DownloadLink, FileLink, ItemConcreteValue, PortLink + +log = logging.getLogger(__name__) + + +async def get_value_from_link( + key: str, + value: PortLink, + fileToKeyMap: Dict, + node_port_creator: Callable[[str], Coroutine[Any, Any, Any]], +) -> ItemConcreteValue: + log.debug("Getting value %s", value) + # create a node ports for the other node + other_nodeports = await node_port_creator(value.node_uuid) + # get the port value through that guy + log.debug("Received node from DB %s, now returning value", other_nodeports) + + value = await other_nodeports.get(value.output) + if isinstance(value, Path): + file_name = value.name + # move the file to the right final location + # if a file alias is present use it + if fileToKeyMap: + file_name = next(iter(fileToKeyMap)) + + file_path = data_items_utils.create_file_path(key, file_name) + if value == file_path: + # this can happen in case + return value + if file_path.exists(): + file_path.unlink() + file_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(value), str(file_path)) + value = file_path + + return value + + +async def pull_file_from_store(key: str, fileToKeyMap: Dict, value: FileLink) -> Path: + log.debug("Getting value from storage %s", value) + # do not make any assumption about s3_path, it is a str containing stuff that can be anything depending on the store + local_path = data_items_utils.create_folder_path(key) + downloaded_file = await filemanager.download_file_from_s3( + store_id=value.store, s3_object=value.path, local_folder=local_path + ) + # if a file alias is present use it to rename the file accordingly + if fileToKeyMap: + renamed_file = local_path / next(iter(fileToKeyMap)) + if downloaded_file != renamed_file: + if renamed_file.exists(): + renamed_file.unlink() + shutil.move(downloaded_file, renamed_file) + downloaded_file = renamed_file + + return downloaded_file + + +async def push_file_to_store(file: Path) -> FileLink: + log.debug("file path %s will be uploaded to s3", file) + s3_object = data_items_utils.encode_file_id( + file, project_id=config.PROJECT_ID, node_id=config.NODE_UUID + ) + store_id = await filemanager.upload_file( + store_name=config.STORE, s3_object=s3_object, local_file_path=file + ) + log.debug("file path %s uploaded", file) + return FileLink(store=store_id, path=s3_object) + + +async def pull_file_from_download_link( + key: str, fileToKeyMap: Dict, value: DownloadLink +) -> Path: + log.debug( + "Getting value from download link [%s] with label %s", + value["downloadLink"], + value.get("label", "undef"), + ) + + download_link = URL(value["downloadLink"]) + local_path = data_items_utils.create_folder_path(key) + downloaded_file = await filemanager.download_file_from_link( + download_link, + local_path, + file_name=next(iter(fileToKeyMap)) if fileToKeyMap else None, + ) + + return downloaded_file + + +def is_file_type(port_type: str): + return port_type.startswith("data:") diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py new file mode 100644 index 000000000000..1ff7d116ed3c --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -0,0 +1,76 @@ +import json +from typing import List + +from aiopg.sa.result import RowProxy + +from ..node_ports.dbmanager import DBManager +from ..node_ports.exceptions import InvalidProtocolError +from .nodeports_v2 import Nodeports + +NODE_REQUIORED_KEYS: List[str] = { + "schema", + "inputs", + "outputs", +} + + +async def create_nodeports_from_db( + db_manager: DBManager, node_uuid: str, auto_update: bool = False +) -> Nodeports: + """creates a nodeport object from a row from comp_tasks""" + row: RowProxy = await db_manager.get_ports_configuration_from_node_uuid(node_uuid) + port_cfg = json.loads(row) + if any(k not in port_cfg for k in NODE_REQUIORED_KEYS): + raise InvalidProtocolError( + port_cfg, "nodeport in comp_task does not follow protocol" + ) + # convert to our internal node ports + _PY_INT = "__root__" + node_ports_cfg = {"inputs": {_PY_INT: {}}, "outputs": {_PY_INT: {}}} + for port_type in ["inputs", "outputs"]: + # schemas first + node_ports_cfg.update( + { + port_type: {_PY_INT: port_cfg["schema"][port_type]}, + } + ) + # add the key and the payload + for key, port_value in node_ports_cfg[port_type][_PY_INT].items(): + port_value["key"] = key + port_value["value"] = port_cfg[port_type].get(key, None) + + ports = Nodeports( + **node_ports_cfg, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=save_nodeports_to_db, + node_port_creator_cb=create_nodeports_from_db, + auto_update=auto_update + ) + return ports + + +async def save_nodeports_to_db(nodeports: Nodeports): + _nodeports_cfg = nodeports.dict( + include={"internal_inputs", "internal_outputs"}, + by_alias=True, + exclude_unset=True, + ) + + # convert to DB + port_cfg = {"schema": {"inputs": {}, "outputs": {}}, "inputs": {}, "outputs": {}} + for port_type in ["inputs", "outputs"]: + for port_key, port_values in _nodeports_cfg[port_type].items(): + # schemas + key_schema = { + k: v + for k, v in _nodeports_cfg[port_type][port_key].items() + if k not in ["key", "value"] + } + port_cfg["schema"][port_type][port_key] = key_schema + # payload + port_cfg[port_type][port_key] = port_values["value"] + + await nodeports.db_manager.write_ports_configuration( + json.dumps(port_cfg), nodeports.node_uuid + ) diff --git a/packages/simcore-sdk/tests/integration/test_nodeports2.py b/packages/simcore-sdk/tests/integration/test_nodeports2.py new file mode 100644 index 000000000000..9366762a6022 --- /dev/null +++ b/packages/simcore-sdk/tests/integration/test_nodeports2.py @@ -0,0 +1,448 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +# pylint:disable=too-many-arguments +# pylint:disable=pointless-statement + +import filecmp +import tempfile +from pathlib import Path +from typing import Callable, Dict, Type + +import np_helpers # pylint: disable=no-name-in-module +import pytest +import sqlalchemy as sa +from simcore_sdk import node_ports_v2 +from simcore_sdk.node_ports import exceptions +from simcore_sdk.node_ports_v2.links import ItemConcreteValue +from simcore_sdk.node_ports_v2.nodeports_v2 import Nodeports + +core_services = ["postgres", "storage"] + +ops_services = ["minio"] + + +async def _check_port_valid( + ports: Nodeports, config_dict: Dict, port_type: str, key_name: str, key: str +): + assert (await getattr(ports, port_type))[key].key == key_name + # check required values + assert (await getattr(ports, port_type))[key].label == config_dict["schema"][ + port_type + ][key_name]["label"] + assert (await getattr(ports, port_type))[key].description == config_dict["schema"][ + port_type + ][key_name]["description"] + assert (await getattr(ports, port_type))[key].type == config_dict["schema"][ + port_type + ][key_name]["type"] + assert (await getattr(ports, port_type))[key].displayOrder == config_dict["schema"][ + port_type + ][key_name]["displayOrder"] + # check optional values + if "defaultValue" in config_dict["schema"][port_type][key_name]: + assert (await getattr(ports, port_type))[key].defaultValue == config_dict[ + "schema" + ][port_type][key_name]["defaultValue"] + else: + assert (await getattr(ports, port_type))[key].defaultValue == None + if "fileToKeyMap" in config_dict["schema"][port_type][key_name]: + assert (await getattr(ports, port_type))[key].fileToKeyMap == config_dict[ + "schema" + ][port_type][key_name]["fileToKeyMap"] + else: + assert (await getattr(ports, port_type))[key].fileToKeyMap == None + if "widget" in config_dict["schema"][port_type][key_name]: + assert (await getattr(ports, port_type))[key].widget == config_dict["schema"][ + port_type + ][key_name]["widget"] + else: + assert (await getattr(ports, port_type))[key].widget == None + # check payload values + if key_name in config_dict[port_type]: + if isinstance(config_dict[port_type][key_name], dict): + assert (await getattr(ports, port_type))[key].value.dict( + by_alias=True, exclude_unset=True + ) == config_dict[port_type][key_name] + else: + assert (await getattr(ports, port_type))[key].value == config_dict[ + port_type + ][key_name] + elif "defaultValue" in config_dict["schema"][port_type][key_name]: + assert (await getattr(ports, port_type))[key].value == config_dict["schema"][ + port_type + ][key_name]["defaultValue"] + else: + assert (await getattr(ports, port_type))[key].value == None + + +async def _check_ports_valid(ports: Nodeports, config_dict: Dict, port_type: str): + for key in config_dict["schema"][port_type].keys(): + # test using "key" name + await _check_port_valid(ports, config_dict, port_type, key, key) + # test using index + key_index = list(config_dict["schema"][port_type].keys()).index(key) + await _check_port_valid(ports, config_dict, port_type, key, key_index) + + +async def check_config_valid(ports: Nodeports, config_dict: Dict): + await _check_ports_valid(ports, config_dict, "inputs") + await _check_ports_valid(ports, config_dict, "outputs") + + +async def test_default_configuration( + default_configuration: Dict, +): # pylint: disable=W0613, W0621 + config_dict = default_configuration + await check_config_valid(await node_ports_v2.ports(), config_dict) + + +async def test_invalid_ports(special_configuration: Callable): + config_dict, _, _ = special_configuration() + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + + with pytest.raises(exceptions.UnboundPortError): + (await PORTS.inputs)[0] + + with pytest.raises(exceptions.UnboundPortError): + (await PORTS.outputs)[0] + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype", + [ + ("integer", 26, int), + ("integer", 0, int), + ("integer", -52, int), + ("number", -746.4748, float), + ("number", 0.0, float), + ("number", 4566.11235, float), + ("boolean", False, bool), + ("boolean", True, bool), + ("string", "test-string", str), + ("string", "", str), + ], +) +async def test_port_value_accessors( + special_configuration: Callable, + item_type: str, + item_value: ItemConcreteValue, + item_pytype: Type, +): # pylint: disable=W0613, W0621 + item_key = "some key" + config_dict, _, _ = special_configuration( + inputs=[(item_key, item_type, item_value)], + outputs=[(item_key, item_type, None)], + ) + + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + + assert isinstance(await (await PORTS.inputs)[item_key].get(), item_pytype) + assert await (await PORTS.inputs)[item_key].get() == item_value + assert await (await PORTS.outputs)[item_key].get() is None + + assert isinstance(await PORTS.get(item_key), item_pytype) + assert await PORTS.get(item_key) == item_value + + await (await PORTS.outputs)[item_key].set(item_value) + assert (await PORTS.outputs)[item_key].value == item_value + assert isinstance(await (await PORTS.outputs)[item_key].get(), item_pytype) + assert await (await PORTS.outputs)[item_key].get() == item_value + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype, config_value", + [ + ("data:*/*", __file__, Path, {"store": "0", "path": __file__}), + ("data:text/*", __file__, Path, {"store": "0", "path": __file__}), + ("data:text/py", __file__, Path, {"store": "0", "path": __file__}), + ], +) +async def test_port_file_accessors( + special_configuration: Callable, + filemanager_cfg: None, + s3_simcore_location: str, + bucket: str, + item_type: str, + item_value: str, + item_pytype: Type, + config_value: Dict[str, str], +): # pylint: disable=W0613, W0621 + config_dict, project_id, node_uuid = special_configuration( + inputs=[("in_1", item_type, config_value)], + outputs=[("out_34", item_type, None)], + ) + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + assert await (await PORTS.outputs)["out_34"].get() is None # check emptyness + # with pytest.raises(exceptions.S3InvalidPathError): + # await PORTS.inputs["in_1"].get() + + # this triggers an upload to S3 + configuration change + await (await PORTS.outputs)["out_34"].set(item_value) + # this is the link to S3 storage + assert (await PORTS.outputs)["out_34"].value.dict( + by_alias=True, exclude_unset=True + ) == { + "store": s3_simcore_location, + "path": Path(str(project_id), str(node_uuid), Path(item_value).name).as_posix(), + } + # this triggers a download from S3 to a location in /tempdir/simcorefiles/item_key + assert isinstance(await (await PORTS.outputs)["out_34"].get(), item_pytype) + assert (await (await PORTS.outputs)["out_34"].get()).exists() + assert str(await (await PORTS.outputs)["out_34"].get()).startswith( + str(Path(tempfile.gettempdir(), "simcorefiles", "out_34")) + ) + filecmp.clear_cache() + assert filecmp.cmp(item_value, await (await PORTS.outputs)["out_34"].get()) + + +async def test_adding_new_ports( + special_configuration: Callable, + postgres_session: sa.orm.session.Session, +): + config_dict, project_id, node_uuid = special_configuration() + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + + # replace the configuration now, add an input + config_dict["schema"]["inputs"].update( + { + "in_15": { + "label": "additional data", + "description": "here some additional data", + "displayOrder": 2, + "type": "integer", + } + } + ) + config_dict["inputs"].update({"in_15": 15}) + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + + # # replace the configuration now, add an output + config_dict["schema"]["outputs"].update( + { + "out_15": { + "label": "output data", + "description": "a cool output", + "displayOrder": 2, + "type": "boolean", + } + } + ) + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + + +async def test_removing_ports( + special_configuration: Callable, + postgres_session: sa.orm.session.Session, +): + config_dict, project_id, node_uuid = special_configuration( + inputs=[("in_14", "integer", 15), ("in_17", "boolean", False)], + outputs=[("out_123", "string", "blahblah"), ("out_2", "number", -12.3)], + ) # pylint: disable=W0612 + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + # let's remove the first input + del config_dict["schema"]["inputs"]["in_14"] + del config_dict["inputs"]["in_14"] + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + # let's do the same for the second output + del config_dict["schema"]["outputs"]["out_2"] + del config_dict["outputs"]["out_2"] + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype", + [ + ("integer", 26, int), + ("integer", 0, int), + ("integer", -52, int), + ("number", -746.4748, float), + ("number", 0.0, float), + ("number", 4566.11235, float), + ("boolean", False, bool), + ("boolean", True, bool), + ("string", "test-string", str), + ("string", "", str), + ], +) +async def test_get_value_from_previous_node( + special_2nodes_configuration: Callable, + node_link: Callable, + item_type: str, + item_value: ItemConcreteValue, + item_pytype: Type, +): + config_dict, _, _ = special_2nodes_configuration( + prev_node_outputs=[("output_123", item_type, item_value)], + inputs=[("in_15", item_type, node_link("output_123"))], + ) + PORTS = await node_ports_v2.ports() + + await check_config_valid(PORTS, config_dict) + input_value = await (await PORTS.inputs)["in_15"].get() + assert isinstance(input_value, item_pytype) + assert await (await PORTS.inputs)["in_15"].get() == item_value + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype", + [ + ("data:*/*", __file__, Path), + ("data:text/*", __file__, Path), + ("data:text/py", __file__, Path), + ], +) +async def test_get_file_from_previous_node( + special_2nodes_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + node_link: Callable, + store_link: Callable, + item_type: str, + item_value: str, + item_pytype: Type, +): + config_dict, _, _ = special_2nodes_configuration( + prev_node_outputs=[ + ("output_123", item_type, store_link(item_value, project_id, node_uuid)) + ], + inputs=[("in_15", item_type, node_link("output_123"))], + project_id=project_id, + previous_node_id=node_uuid, + ) + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + file_path = await (await PORTS.inputs)["in_15"].get() + assert isinstance(file_path, item_pytype) + assert file_path == Path( + tempfile.gettempdir(), "simcorefiles", "in_15", Path(item_value).name + ) + assert file_path.exists() + filecmp.clear_cache() + assert filecmp.cmp(file_path, item_value) + + +@pytest.mark.parametrize( + "item_type, item_value, item_alias, item_pytype", + [ + ("data:*/*", __file__, Path(__file__).name, Path), + ("data:*/*", __file__, "some funky name.txt", Path), + ("data:text/*", __file__, "some funky name without extension", Path), + ("data:text/py", __file__, "öä$äö2-34 name without extension", Path), + ], +) +async def test_get_file_from_previous_node_with_mapping_of_same_key_name( + special_2nodes_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + node_link: Callable, + store_link: Callable, + postgres_session: sa.orm.session.Session, + item_type: str, + item_value: str, + item_alias: str, + item_pytype: Type, +): + config_dict, _, this_node_uuid = special_2nodes_configuration( + prev_node_outputs=[ + ("in_15", item_type, store_link(item_value, project_id, node_uuid)) + ], + inputs=[("in_15", item_type, node_link("in_15"))], + project_id=project_id, + previous_node_id=node_uuid, + ) + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + # add a filetokeymap + config_dict["schema"]["inputs"]["in_15"]["fileToKeyMap"] = {item_alias: "in_15"} + np_helpers.update_configuration( + postgres_session, project_id, this_node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + file_path = await (await PORTS.inputs)["in_15"].get() + assert isinstance(file_path, item_pytype) + assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_15", item_alias) + assert file_path.exists() + filecmp.clear_cache() + assert filecmp.cmp(file_path, item_value) + + +@pytest.mark.parametrize( + "item_type, item_value, item_alias, item_pytype", + [ + ("data:*/*", __file__, Path(__file__).name, Path), + ("data:*/*", __file__, "some funky name.txt", Path), + ("data:text/*", __file__, "some funky name without extension", Path), + ("data:text/py", __file__, "öä$äö2-34 name without extension", Path), + ], +) +async def test_file_mapping( + special_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + s3_simcore_location: str, + bucket: str, + store_link: Callable, + postgres_session: sa.orm.session.Session, + item_type: str, + item_value: str, + item_alias: str, + item_pytype: Type, +): + config_dict, project_id, node_uuid = special_configuration( + inputs=[("in_1", item_type, store_link(item_value, project_id, node_uuid))], + outputs=[("out_1", item_type, None)], + project_id=project_id, + node_id=node_uuid, + ) + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + # add a filetokeymap + config_dict["schema"]["inputs"]["in_1"]["fileToKeyMap"] = {item_alias: "in_1"} + config_dict["schema"]["outputs"]["out_1"]["fileToKeyMap"] = {item_alias: "out_1"} + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + file_path = await (await PORTS.inputs)["in_1"].get() + assert isinstance(file_path, item_pytype) + assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_1", item_alias) + + # let's get it a second time to see if replacing works + file_path = await (await PORTS.inputs)["in_1"].get() + assert isinstance(file_path, item_pytype) + assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_1", item_alias) + + # now set + invalid_alias = Path("invalid_alias.fjfj") + with pytest.raises(exceptions.PortNotFound): + await PORTS.set_file_by_keymap(invalid_alias) + + await PORTS.set_file_by_keymap(file_path) + file_id = np_helpers.file_uuid(file_path, project_id, node_uuid) + assert (await PORTS.outputs)["out_1"].value.dict( + by_alias=True, exclude_unset=True + ) == { + "store": s3_simcore_location, + "path": file_id, + } From 6a1d7b9834015e26cd2799936c520fcbc347529a Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 21:41:12 +0100 Subject: [PATCH 11/74] add some unit tests --- .../src/simcore_sdk/node_ports_v2/links.py | 6 +++++- packages/simcore-sdk/tests/unit/test_links.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 packages/simcore-sdk/tests/unit/test_links.py diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py index f6c5e37c2f68..ec9b065ff088 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py @@ -11,9 +11,13 @@ StrictStr, ) +UUID_REGEX = ( + r"^[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" +) + class PortLink(BaseModel): - node_uuid: str = Field(..., alias="nodeUuid") + node_uuid: str = Field(..., regex=UUID_REGEX, alias="nodeUuid") output: str diff --git a/packages/simcore-sdk/tests/unit/test_links.py b/packages/simcore-sdk/tests/unit/test_links.py new file mode 100644 index 000000000000..708e087b11b1 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_links.py @@ -0,0 +1,19 @@ +from typing import Dict +from uuid import uuid4 + +import pytest +from pydantic import ValidationError +from simcore_sdk.node_ports_v2.links import PortLink + + +@pytest.mark.parametrize( + "port_link", + [ + {"nodeUuid": f"{uuid4()}"}, + {"output": "some stuff"}, + {"nodeUuid": "some stuff", "output": "some stuff"}, + ], +) +def test_invalid_port_link(port_link: Dict[str, str]): + with pytest.raises(ValidationError): + PortLink(**port_link) From ddc1fd27dd7d869460708677de635bf72d20afcd Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 21:52:00 +0100 Subject: [PATCH 12/74] more unit tests --- .../src/simcore_sdk/node_ports_v2/links.py | 5 ++-- packages/simcore-sdk/tests/unit/test_links.py | 30 ++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py index ec9b065ff088..1befefd1b8f6 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py @@ -14,11 +14,12 @@ UUID_REGEX = ( r"^[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" ) +PROPERTY_KEY_RE = r"^[-_a-zA-Z0-9]+$" class PortLink(BaseModel): node_uuid: str = Field(..., regex=UUID_REGEX, alias="nodeUuid") - output: str + output: str = Field(..., regex=PROPERTY_KEY_RE) class DownloadLink(BaseModel): @@ -28,7 +29,7 @@ class DownloadLink(BaseModel): class FileLink(BaseModel): store: Union[str, int] - path: str + path: str = Field(..., regex=r".+") dataset: Optional[str] label: Optional[str] diff --git a/packages/simcore-sdk/tests/unit/test_links.py b/packages/simcore-sdk/tests/unit/test_links.py index 708e087b11b1..0ff812af34c9 100644 --- a/packages/simcore-sdk/tests/unit/test_links.py +++ b/packages/simcore-sdk/tests/unit/test_links.py @@ -3,7 +3,7 @@ import pytest from pydantic import ValidationError -from simcore_sdk.node_ports_v2.links import PortLink +from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink @pytest.mark.parametrize( @@ -12,8 +12,36 @@ {"nodeUuid": f"{uuid4()}"}, {"output": "some stuff"}, {"nodeUuid": "some stuff", "output": "some stuff"}, + {"nodeUuid": "", "output": "some stuff"}, + {"nodeUuid": f"{uuid4()}", "output": ""}, ], ) def test_invalid_port_link(port_link: Dict[str, str]): with pytest.raises(ValidationError): PortLink(**port_link) + + +@pytest.mark.parametrize( + "download_link", + [ + {"downloadLink": ""}, + {"downloadLink": "some stuff"}, + {"label": "some stuff"}, + ], +) +def test_invalid_download_link(download_link: Dict[str, str]): + with pytest.raises(ValidationError): + DownloadLink(**download_link) + + +@pytest.mark.parametrize( + "file_link", + [ + {"store": ""}, + {"store": "0", "path": ""}, + {"path": "/somefile/blahblah:"}, + ], +) +def test_invalid_file_link(file_link: Dict[str, str]): + with pytest.raises(ValidationError): + FileLink(**file_link) From ee429bb9d7870426c121a5dc69761ee33e397c6f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 22:28:00 +0100 Subject: [PATCH 13/74] improve types --- .../simcore_sdk/node_ports_v2/constants.py | 7 +++ .../src/simcore_sdk/node_ports_v2/links.py | 7 +-- .../simcore_sdk/node_ports_v2/nodeports_v2.py | 4 +- .../src/simcore_sdk/node_ports_v2/port.py | 25 ++++---- packages/simcore-sdk/tests/unit/test_links.py | 11 +++- packages/simcore-sdk/tests/unit/test_port.py | 59 +++++++++++++++++++ 6 files changed, 93 insertions(+), 20 deletions(-) create mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/constants.py create mode 100644 packages/simcore-sdk/tests/unit/test_port.py diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/constants.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/constants.py new file mode 100644 index 000000000000..d7480c9d5273 --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/constants.py @@ -0,0 +1,7 @@ +UUID_REGEX = ( + r"^[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" +) +PROPERTY_TYPE_RE = r"^(number|integer|boolean|string|data:([^/\s,]+/[^/\s,]+|\[[^/\s,]+/[^/\s,]+(,[^/\s]+/[^/,\s]+)*\]))$" +PROPERTY_KEY_RE = r"^[-_a-zA-Z0-9]+$" + +STORE_PATH_REGEX = r"^.+$" diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py index 1befefd1b8f6..63deb220b5b4 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py @@ -11,10 +11,7 @@ StrictStr, ) -UUID_REGEX = ( - r"^[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" -) -PROPERTY_KEY_RE = r"^[-_a-zA-Z0-9]+$" +from .constants import PROPERTY_KEY_RE, STORE_PATH_REGEX, UUID_REGEX class PortLink(BaseModel): @@ -29,7 +26,7 @@ class DownloadLink(BaseModel): class FileLink(BaseModel): store: Union[str, int] - path: str = Field(..., regex=r".+") + path: str = Field(..., regex=STORE_PATH_REGEX) dataset: Optional[str] label: Optional[str] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 3c6fbba3c906..1cb21023eab8 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -67,8 +67,8 @@ async def set(self, item_key: str, item_value): async def set_file_by_keymap(self, item_value: Path): for output in (await self.outputs).values(): - if is_file_type(output.type) and output.fileToKeyMap: - if item_value.name in output.fileToKeyMap: + if is_file_type(output.type) and output.file_to_key_map: + if item_value.name in output.file_to_key_map: await output.set(item_value) return raise PortNotFound(msg=f"output port for item {item_value} not found") diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index a9015eb0d72f..957a7ecf557a 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -5,6 +5,7 @@ from pydantic import ( BaseModel, + Field, PrivateAttr, StrictBool, StrictFloat, @@ -14,11 +15,11 @@ from ..node_ports.exceptions import InvalidItemTypeError, UnboundPortError from . import port_utils +from .constants import PROPERTY_KEY_RE, PROPERTY_TYPE_RE from .links import DataItemValue, DownloadLink, FileLink, ItemConcreteValue, PortLink log = logging.getLogger(__name__) - TYPE_TO_PYTYPE: Dict[str, Type[ItemConcreteValue]] = { "integer": int, "number": float, @@ -28,13 +29,15 @@ class Port(BaseModel): - key: str + key: str = Field(..., regex=PROPERTY_KEY_RE) label: str description: str - type: str - displayOrder: float - fileToKeyMap: Optional[Dict[str, str]] = None - defaultValue: Optional[Union[StrictBool, StrictInt, StrictFloat, str]] = None + type: str = Field(..., regex=PROPERTY_TYPE_RE) + display_order: float = Field(..., alias="displayOrder") + file_to_key_map: Optional[Dict[str, str]] = Field(None, alias="fileToKeyMap") + default_value: Optional[Union[StrictBool, StrictInt, StrictFloat, str]] = Field( + None, alias="defaultValue" + ) widget: Optional[Dict] = None value: Optional[DataItemValue] @@ -46,8 +49,8 @@ class Port(BaseModel): @validator("value", always=True) @classmethod def ensure_value(cls, v, values): - if not v and values.get("defaultValue"): - return values["defaultValue"] + if not v and values.get("default_value"): + return values["default_value"] return v def __init__(self, **data): @@ -76,18 +79,18 @@ async def get(self) -> ItemConcreteValue: value = await port_utils.get_value_from_link( self.key, self.value, - self.fileToKeyMap, + self.file_to_key_map, self._node_ports._node_ports_creator_cb, ) elif isinstance(self.value, FileLink): # this is a link from storage value = await port_utils.pull_file_from_store( - self.key, self.fileToKeyMap, self.value + self.key, self.file_to_key_map, self.value ) elif isinstance(self.value, DownloadLink): # this is a downloadable link value = await port_utils.pull_file_from_download_link( - self.key, self.fileToKeyMap, self.value + self.key, self.file_to_key_map, self.value ) else: # this is directly the value diff --git a/packages/simcore-sdk/tests/unit/test_links.py b/packages/simcore-sdk/tests/unit/test_links.py index 0ff812af34c9..5116311ae01a 100644 --- a/packages/simcore-sdk/tests/unit/test_links.py +++ b/packages/simcore-sdk/tests/unit/test_links.py @@ -6,14 +6,21 @@ from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink +def test_valid_port_link(): + port_link = {"nodeUuid": f"{uuid4()}", "output": "some_key"} + PortLink(**port_link) + + @pytest.mark.parametrize( "port_link", [ {"nodeUuid": f"{uuid4()}"}, - {"output": "some stuff"}, - {"nodeUuid": "some stuff", "output": "some stuff"}, + {"output": "some_stuff"}, + {"nodeUuid": "some stuff", "output": "some_stuff"}, {"nodeUuid": "", "output": "some stuff"}, {"nodeUuid": f"{uuid4()}", "output": ""}, + {"nodeUuid": f"{uuid4()}", "output": "some.key"}, + {"nodeUuid": f"{uuid4()}", "output": "some:key"}, ], ) def test_invalid_port_link(port_link: Dict[str, str]): diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py new file mode 100644 index 000000000000..4e11fb7ff58e --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -0,0 +1,59 @@ +from typing import Any, Dict + +import pytest +from pydantic.error_wrappers import ValidationError +from simcore_sdk.node_ports_v2.port import Port + + +@pytest.mark.parametrize( + "port_cfg", + [ + { + "key": "some_key", + "label": "some label", + "description": "some description", + "type": "integer", + "displayOrder": 2.3, + }, + { + "key": "some_key", + "label": "", + "description": "", + "type": "integer", + "displayOrder": 2.3, + }, + ], +) +def test_valid_port(port_cfg: Dict[str, Any]): + Port(**port_cfg) + + +@pytest.mark.parametrize( + "port_cfg", + [ + { + "key": "some.key", + "label": "some label", + "description": "some description", + "type": "integer", + "displayOrder": 2.3, + }, + { + "key": "some:key", + "label": "", + "description": "", + "type": "integer", + "displayOrder": 2.3, + }, + { + "key": "some_key", + "label": "", + "description": "", + "type": "blahblah", + "displayOrder": 2.3, + }, + ], +) +def test_invalid_port(port_cfg: Dict[str, Any]): + with pytest.raises(ValidationError): + Port(**port_cfg) From 7a960ef31986f140efbedff9b462b4ee642be502 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 22:39:04 +0100 Subject: [PATCH 14/74] use pydantic with email validator --- packages/simcore-sdk/requirements/_base.in | 2 +- packages/simcore-sdk/requirements/_test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in index cd560ad2a6bb..6cc3d313f020 100644 --- a/packages/simcore-sdk/requirements/_base.in +++ b/packages/simcore-sdk/requirements/_base.in @@ -12,7 +12,7 @@ aiohttp aiopg[sa] networkx psycopg2-binary -pydantic +pydantic[email] tenacity trafaret-config diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index d2925688019d..ed1163967993 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -48,7 +48,7 @@ python-dateutil==2.8.1 # via alembic python-dotenv==0.15.0 # via -r requirements/_test.in python-editor==1.0.4 # via alembic requests==2.25.0 # via -r requirements/_test.in, coveralls, docker -six==1.15.0 # via -c requirements/_base.txt, astroid, docker, packaging, python-dateutil, websocket-client +six==1.15.0 # via -c requirements/_base.txt, astroid, docker, python-dateutil, websocket-client sqlalchemy[postgresql_psycopg2binary]==1.3.20 # via -c requirements/_base.txt, alembic termcolor==1.1.0 # via pytest-sugar toml==0.10.2 # via pylint, pytest From 27fbeba1e487287e2aee36ecc4ee89bbef16ca73 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 22:46:59 +0100 Subject: [PATCH 15/74] re-use classes from models library --- .../src/models_library/services.py | 15 +++++- .../simcore_sdk/node_ports_v2/constants.py | 7 --- .../src/simcore_sdk/node_ports_v2/links.py | 38 ------------- .../simcore_sdk/node_ports_v2/nodeports_v2.py | 2 +- .../src/simcore_sdk/node_ports_v2/port.py | 53 ++++++++++++------- 5 files changed, 48 insertions(+), 67 deletions(-) delete mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/constants.py delete mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index 99dbfc7718d7..73c89213dd65 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -6,7 +6,18 @@ from enum import Enum from typing import Dict, List, Optional, Union -from pydantic import BaseModel, EmailStr, Extra, Field, HttpUrl, constr, validator +from pydantic import ( + BaseModel, + EmailStr, + Extra, + Field, + HttpUrl, + StrictBool, + StrictFloat, + StrictInt, + constr, + validator, +) from pydantic.types import PositiveInt from .basic_regex import VERSION_RE @@ -150,7 +161,7 @@ class ServiceProperty(BaseModel): description="Place the data associated with the named keys in files", examples=[{"dir/input1.txt": "key_1", "dir33/input2.txt": "key2"}], ) - default_value: Optional[Union[str, float, bool, int]] = Field( + default_value: Optional[Union[StrictBool, StrictInt, StrictFloat, str]] = Field( None, alias="defaultValue", examples=["Dog", True] ) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/constants.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/constants.py deleted file mode 100644 index d7480c9d5273..000000000000 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/constants.py +++ /dev/null @@ -1,7 +0,0 @@ -UUID_REGEX = ( - r"^[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$" -) -PROPERTY_TYPE_RE = r"^(number|integer|boolean|string|data:([^/\s,]+/[^/\s,]+|\[[^/\s,]+/[^/\s,]+(,[^/\s]+/[^/,\s]+)*\]))$" -PROPERTY_KEY_RE = r"^[-_a-zA-Z0-9]+$" - -STORE_PATH_REGEX = r"^.+$" diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py deleted file mode 100644 index 63deb220b5b4..000000000000 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py +++ /dev/null @@ -1,38 +0,0 @@ -from pathlib import Path -from typing import Optional, Union - -from pydantic import ( - AnyUrl, - BaseModel, - Field, - StrictBool, - StrictFloat, - StrictInt, - StrictStr, -) - -from .constants import PROPERTY_KEY_RE, STORE_PATH_REGEX, UUID_REGEX - - -class PortLink(BaseModel): - node_uuid: str = Field(..., regex=UUID_REGEX, alias="nodeUuid") - output: str = Field(..., regex=PROPERTY_KEY_RE) - - -class DownloadLink(BaseModel): - download_link: AnyUrl = Field(..., alias="downloadLink") - label: Optional[str] - - -class FileLink(BaseModel): - store: Union[str, int] - path: str = Field(..., regex=STORE_PATH_REGEX) - dataset: Optional[str] - label: Optional[str] - - -DataItemValue = Union[ - StrictBool, StrictInt, StrictFloat, StrictStr, DownloadLink, PortLink, FileLink -] - -ItemConcreteValue = Union[int, float, bool, str, Path] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 1cb21023eab8..0aa8df5ccefe 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -67,7 +67,7 @@ async def set(self, item_key: str, item_value): async def set_file_by_keymap(self, item_value: Path): for output in (await self.outputs).values(): - if is_file_type(output.type) and output.file_to_key_map: + if is_file_type(output.property_type) and output.file_to_key_map: if item_value.name in output.file_to_key_map: await output.set(item_value) return diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 957a7ecf557a..476750194b13 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -3,23 +3,39 @@ from pprint import pformat from typing import Dict, Optional, Tuple, Type, Union +from models_library.projects_nodes_io import BaseFileLink, DownloadLink, PortLink +from models_library.services import PROPERTY_KEY_RE, ServiceProperty from pydantic import ( BaseModel, + Extra, Field, PrivateAttr, StrictBool, StrictFloat, StrictInt, + StrictStr, validator, ) from ..node_ports.exceptions import InvalidItemTypeError, UnboundPortError from . import port_utils -from .constants import PROPERTY_KEY_RE, PROPERTY_TYPE_RE -from .links import DataItemValue, DownloadLink, FileLink, ItemConcreteValue, PortLink log = logging.getLogger(__name__) + +class FileLink(BaseFileLink): + """ allow all kind of file links """ + + class Config: + extra = Extra.allow + + +DataItemValue = Union[ + StrictBool, StrictInt, StrictFloat, StrictStr, DownloadLink, PortLink, FileLink +] + +ItemConcreteValue = Union[int, float, bool, str, Path] + TYPE_TO_PYTYPE: Dict[str, Type[ItemConcreteValue]] = { "integer": int, "number": float, @@ -28,16 +44,8 @@ } -class Port(BaseModel): +class Port(ServiceProperty): key: str = Field(..., regex=PROPERTY_KEY_RE) - label: str - description: str - type: str = Field(..., regex=PROPERTY_TYPE_RE) - display_order: float = Field(..., alias="displayOrder") - file_to_key_map: Optional[Dict[str, str]] = Field(None, alias="fileToKeyMap") - default_value: Optional[Union[StrictBool, StrictInt, StrictFloat, str]] = Field( - None, alias="defaultValue" - ) widget: Optional[Dict] = None value: Optional[DataItemValue] @@ -58,16 +66,21 @@ def __init__(self, **data): # let's define the converter self._py_value_type = ( (Path, str) - if port_utils.is_file_type(self.type) - else (TYPE_TO_PYTYPE[self.type]) + if port_utils.is_file_type(self.property_type) + else (TYPE_TO_PYTYPE[self.property_type]) ) self._py_value_converter = ( - Path if port_utils.is_file_type(self.type) else TYPE_TO_PYTYPE[self.type] + Path + if port_utils.is_file_type(self.property_type) + else TYPE_TO_PYTYPE[self.property_type] ) async def get(self) -> ItemConcreteValue: log.debug( - "getting %s[%s] with value %s", self.key, self.type, pformat(self.value) + "getting %s[%s] with value %s", + self.key, + self.property_type, + pformat(self.value), ) if self.value is None: @@ -99,15 +112,17 @@ async def get(self) -> ItemConcreteValue: return self._py_value_converter(value) async def set(self, new_value: ItemConcreteValue): - log.debug("setting %s[%s] with value %s", self.key, self.type, new_value) + log.debug( + "setting %s[%s] with value %s", self.key, self.property_type, new_value + ) if not isinstance(new_value, self._py_value_type): - raise InvalidItemTypeError(self.type, new_value) + raise InvalidItemTypeError(self.property_type, new_value) # convert the concrete value to a data value data_value = self._py_value_converter(new_value) - if port_utils.is_file_type(self.type): + if port_utils.is_file_type(self.property_type): if not data_value.exists() or not data_value.is_file(): - raise InvalidItemTypeError(self.type, new_value) + raise InvalidItemTypeError(self.property_type, new_value) data_value: FileLink = await port_utils.push_file_to_store(data_value) self.value = data_value From c09e7395edb7a1c2bc8053df5b95603bf2b50729 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 22:55:46 +0100 Subject: [PATCH 16/74] use model library as base --- .../src/models_library/projects_nodes_io.py | 2 +- .../src/simcore_sdk/node_ports_v2/links.py | 21 +++++++++++++++ .../simcore_sdk/node_ports_v2/nodeports_v2.py | 3 +-- .../src/simcore_sdk/node_ports_v2/port.py | 27 ++----------------- 4 files changed, 25 insertions(+), 28 deletions(-) create mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py diff --git a/packages/models-library/src/models_library/projects_nodes_io.py b/packages/models-library/src/models_library/projects_nodes_io.py index 305176b8ad3c..4836a2854955 100644 --- a/packages/models-library/src/models_library/projects_nodes_io.py +++ b/packages/models-library/src/models_library/projects_nodes_io.py @@ -9,7 +9,6 @@ from .services import PROPERTY_KEY_RE - NodeID = UUID # Pydantic does not support exporting a jsonschema with Dict keys being something else than a str @@ -54,6 +53,7 @@ class BaseFileLink(BaseModel): ) path: str = Field( ..., + regex=r"^.+$", description="The path to the file in the storage provider domain", example=[ "N:package:b05739ef-260c-4038-b47d-0240d04b0599", diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py new file mode 100644 index 000000000000..61078dd01aa6 --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py @@ -0,0 +1,21 @@ +from pathlib import Path +from typing import Union + +from models_library.projects_nodes_io import BaseFileLink, DownloadLink, PortLink +from pydantic import Extra, StrictBool, StrictFloat, StrictInt, StrictStr + + +class FileLink(BaseFileLink): + """ allow all kind of file links """ + + class Config: + extra = Extra.allow + + +DataItemValue = Union[ + StrictBool, StrictInt, StrictFloat, StrictStr, DownloadLink, PortLink, FileLink +] + +ItemConcreteValue = Union[int, float, bool, str, Path] + +__all__ = ["FileLink", "DownloadLink", "PortLink", "DataItemValue", "ItemConcreteValue"] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 0aa8df5ccefe..82b805cefafa 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -6,8 +6,7 @@ from ..node_ports.dbmanager import DBManager from ..node_ports.exceptions import PortNotFound, UnboundPortError -from .links import ItemConcreteValue -from .port import InputsList, OutputsList +from .port import InputsList, ItemConcreteValue, OutputsList from .port_utils import is_file_type log = logging.getLogger(__name__) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 476750194b13..84fdc626610d 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -3,39 +3,16 @@ from pprint import pformat from typing import Dict, Optional, Tuple, Type, Union -from models_library.projects_nodes_io import BaseFileLink, DownloadLink, PortLink from models_library.services import PROPERTY_KEY_RE, ServiceProperty -from pydantic import ( - BaseModel, - Extra, - Field, - PrivateAttr, - StrictBool, - StrictFloat, - StrictInt, - StrictStr, - validator, -) +from pydantic import BaseModel, Field, PrivateAttr, validator from ..node_ports.exceptions import InvalidItemTypeError, UnboundPortError from . import port_utils +from .links import DataItemValue, DownloadLink, FileLink, ItemConcreteValue, PortLink log = logging.getLogger(__name__) -class FileLink(BaseFileLink): - """ allow all kind of file links """ - - class Config: - extra = Extra.allow - - -DataItemValue = Union[ - StrictBool, StrictInt, StrictFloat, StrictStr, DownloadLink, PortLink, FileLink -] - -ItemConcreteValue = Union[int, float, bool, str, Path] - TYPE_TO_PYTYPE: Dict[str, Type[ItemConcreteValue]] = { "integer": int, "number": float, From 17647b638e312aee1b9a97a314a3a0f2238910da Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 23:27:24 +0100 Subject: [PATCH 17/74] fix uuid --- .../integration/mock/default_config.json | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/simcore-sdk/tests/integration/mock/default_config.json b/packages/simcore-sdk/tests/integration/mock/default_config.json index b0508a717308..392c27aab8c8 100644 --- a/packages/simcore-sdk/tests/integration/mock/default_config.json +++ b/packages/simcore-sdk/tests/integration/mock/default_config.json @@ -1,60 +1,60 @@ { - "version":"0.1", + "version": "0.1", "schema": { "inputs": { - "in_1":{ + "in_1": { "displayOrder": 0, "label": "computational data", "description": "these are computed data out of a pipeline", "type": "data:*/*", "defaultValue": null, "fileToKeyMap": { - "input1.txt":"in_1" + "input1.txt": "in_1" }, "widget": null }, - "in_5":{ + "in_5": { "displayOrder": 2, "label": "some number", "description": "numbering things", "type": "integer", "defaultValue": 666, - "fileToKeyMap":{}, + "fileToKeyMap": {}, "widget": null } }, - "outputs" : { + "outputs": { "out_1": { - "displayOrder":0, + "displayOrder": 0, "label": "some boolean output", "description": "could be true or false...", "type": "boolean", "defaultValue": null, - "fileToKeyMap":{}, + "fileToKeyMap": {}, "widget": null }, "out_2": { - "displayOrder":1, + "displayOrder": 1, "label": "some file output", "description": "could be anything...", "type": "data:*/*", "defaultValue": null, - "fileToKeyMap":{}, + "fileToKeyMap": {}, "widget": null } } }, - "inputs": { + "inputs": { "in_1": { - "nodeUuid":"456465-45ffd", + "nodeUuid": "793bc431-b935-4f9f-8b04-e4117640e9b6", "output": "outFile" } }, "outputs": { "out_1": false, "out_2": { - "store":"z43-s3", + "store": "z43-s3", "path": "/simcore/outputControllerOut.dat" } } -} \ No newline at end of file +} From 1376d126bc638cfa5e0ab34e9f1590e55df88b98 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 23:27:45 +0100 Subject: [PATCH 18/74] PortLink without UUID --- .../simcore-sdk/src/simcore_sdk/node_ports_v2/links.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py index 61078dd01aa6..572ba9f210f9 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py @@ -1,8 +1,13 @@ from pathlib import Path from typing import Union -from models_library.projects_nodes_io import BaseFileLink, DownloadLink, PortLink -from pydantic import Extra, StrictBool, StrictFloat, StrictInt, StrictStr +from models_library.projects_nodes_io import UUID_REGEX, BaseFileLink, DownloadLink +from models_library.projects_nodes_io import PortLink as BasePortLink +from pydantic import Extra, Field, StrictBool, StrictFloat, StrictInt, StrictStr + + +class PortLink(BasePortLink): + node_uuid: str = Field(..., regex=UUID_REGEX, alias="nodeUuid") class FileLink(BaseFileLink): From b78d1d8e61dc55a2c22d0d0aede87338a897f39d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Thu, 3 Dec 2020 23:27:59 +0100 Subject: [PATCH 19/74] adjust test --- .../tests/integration/test_nodeports2.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/simcore-sdk/tests/integration/test_nodeports2.py b/packages/simcore-sdk/tests/integration/test_nodeports2.py index 9366762a6022..5a8d6fbf2c26 100644 --- a/packages/simcore-sdk/tests/integration/test_nodeports2.py +++ b/packages/simcore-sdk/tests/integration/test_nodeports2.py @@ -33,25 +33,25 @@ async def _check_port_valid( assert (await getattr(ports, port_type))[key].description == config_dict["schema"][ port_type ][key_name]["description"] - assert (await getattr(ports, port_type))[key].type == config_dict["schema"][ - port_type - ][key_name]["type"] - assert (await getattr(ports, port_type))[key].displayOrder == config_dict["schema"][ - port_type - ][key_name]["displayOrder"] + assert (await getattr(ports, port_type))[key].property_type == config_dict[ + "schema" + ][port_type][key_name]["type"] + assert (await getattr(ports, port_type))[key].display_order == config_dict[ + "schema" + ][port_type][key_name]["displayOrder"] # check optional values if "defaultValue" in config_dict["schema"][port_type][key_name]: - assert (await getattr(ports, port_type))[key].defaultValue == config_dict[ + assert (await getattr(ports, port_type))[key].default_value == config_dict[ "schema" ][port_type][key_name]["defaultValue"] else: - assert (await getattr(ports, port_type))[key].defaultValue == None + assert (await getattr(ports, port_type))[key].default_value == None if "fileToKeyMap" in config_dict["schema"][port_type][key_name]: - assert (await getattr(ports, port_type))[key].fileToKeyMap == config_dict[ + assert (await getattr(ports, port_type))[key].file_to_key_map == config_dict[ "schema" ][port_type][key_name]["fileToKeyMap"] else: - assert (await getattr(ports, port_type))[key].fileToKeyMap == None + assert (await getattr(ports, port_type))[key].file_to_key_map == None if "widget" in config_dict["schema"][port_type][key_name]: assert (await getattr(ports, port_type))[key].widget == config_dict["schema"][ port_type @@ -130,7 +130,7 @@ async def test_port_value_accessors( item_value: ItemConcreteValue, item_pytype: Type, ): # pylint: disable=W0613, W0621 - item_key = "some key" + item_key = "some_key" config_dict, _, _ = special_configuration( inputs=[(item_key, item_type, item_value)], outputs=[(item_key, item_type, None)], From ef3ffc15951477c62b9dacf5505a5e9261fa9d50 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 09:21:05 +0100 Subject: [PATCH 20/74] test possible port options --- .../src/simcore_sdk/node_ports_v2/port.py | 2 +- packages/simcore-sdk/tests/unit/test_port.py | 132 +++++++++++++++--- 2 files changed, 115 insertions(+), 19 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 84fdc626610d..e94978707bf4 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -34,7 +34,7 @@ class Port(ServiceProperty): @validator("value", always=True) @classmethod def ensure_value(cls, v, values): - if not v and values.get("default_value"): + if v is None and values.get("default_value"): return values["default_value"] return v diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index 4e11fb7ff58e..b02adc5d91a2 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -1,31 +1,127 @@ -from typing import Any, Dict +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +# pylint:disable=no-member +# pylint:disable=protected-access +import re +from pathlib import Path +from typing import Any, Dict, Type, Union import pytest from pydantic.error_wrappers import ValidationError from simcore_sdk.node_ports_v2.port import Port +def camel_to_snake(name): + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() + + @pytest.mark.parametrize( - "port_cfg", + "port_cfg, exp_value_type, exp_value_converter", [ - { - "key": "some_key", - "label": "some label", - "description": "some description", - "type": "integer", - "displayOrder": 2.3, - }, - { - "key": "some_key", - "label": "", - "description": "", - "type": "integer", - "displayOrder": 2.3, - }, + ( + { + "key": "some_integer", + "label": "some label", + "description": "some description", + "type": "integer", + "displayOrder": 2.3, + "defaultValue": 3, + }, + (int), + int, + ), + ( + { + "key": "some_number", + "label": "", + "description": "", + "type": "number", + "displayOrder": 2.3, + "defaultValue": -23.45, + }, + (float), + float, + ), + ( + { + "key": "some_boolean", + "label": "", + "description": "", + "type": "boolean", + "displayOrder": 2.3, + "defaultValue": True, + }, + (bool), + bool, + ), + ( + { + "key": "some_boolean", + "label": "", + "description": "", + "type": "boolean", + "displayOrder": 2.3, + "defaultValue": True, + "value": False, + }, + (bool), + bool, + ), + ( + { + "key": "some_file", + "label": "", + "description": "", + "type": "data:*/*", + "displayOrder": 2.3, + }, + (Path, str), + Path, + ), + ( + { + "key": "some_file", + "label": "", + "description": "", + "type": "data:*/*", + "displayOrder": 2.3, + "value": __file__, + }, + (Path, str), + Path, + ), ], ) -def test_valid_port(port_cfg: Dict[str, Any]): - Port(**port_cfg) +async def test_valid_port( + port_cfg: Dict[str, Any], + exp_value_type: Type[Union[int, float, bool, str, Path]], + exp_value_converter: Type[Union[int, float, bool, str, Path]], +): + port = Port(**port_cfg) + + for k, v in port_cfg.items(): + camel_key = camel_to_snake(k) + if k == "type": + camel_key = "property_type" + assert v == getattr(port, camel_key) + + assert port._py_value_type == exp_value_type + assert port._py_value_converter == exp_value_converter + + expected_value = None + if "defaultValue" in port_cfg and ( + "value" not in port_cfg or port_cfg["value"] is None + ): + expected_value = port_cfg["defaultValue"] + elif "value" in port_cfg: + expected_value = port_cfg["value"] + assert port.value == expected_value + if expected_value: + assert await port.get() == exp_value_converter(expected_value) + + value = await port.get() @pytest.mark.parametrize( From b2df427792482bd3cfa4af3f7129607e7e6c5e14 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 09:49:41 +0100 Subject: [PATCH 21/74] test port options --- packages/simcore-sdk/tests/unit/test_port.py | 36 ++++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index b02adc5d91a2..b77b349bd6a7 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -18,7 +18,7 @@ def camel_to_snake(name): @pytest.mark.parametrize( - "port_cfg, exp_value_type, exp_value_converter", + "port_cfg, exp_value_type, exp_value_converter, exp_value", [ ( { @@ -31,6 +31,7 @@ def camel_to_snake(name): }, (int), int, + 3, ), ( { @@ -43,6 +44,7 @@ def camel_to_snake(name): }, (float), float, + -23.46, ), ( { @@ -55,6 +57,7 @@ def camel_to_snake(name): }, (bool), bool, + True, ), ( { @@ -68,6 +71,7 @@ def camel_to_snake(name): }, (bool), bool, + False, ), ( { @@ -79,18 +83,20 @@ def camel_to_snake(name): }, (Path, str), Path, + None, ), ( { - "key": "some_file", + "key": "some_file_with_file_in_defaulvalue", "label": "", "description": "", "type": "data:*/*", "displayOrder": 2.3, - "value": __file__, + "defaultValue": __file__, }, (Path, str), Path, + None, ), ], ) @@ -98,6 +104,7 @@ async def test_valid_port( port_cfg: Dict[str, Any], exp_value_type: Type[Union[int, float, bool, str, Path]], exp_value_converter: Type[Union[int, float, bool, str, Path]], + exp_value: Union[int, float, bool, str, Path], ): port = Port(**port_cfg) @@ -110,18 +117,9 @@ async def test_valid_port( assert port._py_value_type == exp_value_type assert port._py_value_converter == exp_value_converter - expected_value = None - if "defaultValue" in port_cfg and ( - "value" not in port_cfg or port_cfg["value"] is None - ): - expected_value = port_cfg["defaultValue"] - elif "value" in port_cfg: - expected_value = port_cfg["value"] - assert port.value == expected_value - if expected_value: - assert await port.get() == exp_value_converter(expected_value) - - value = await port.get() + assert port.value == exp_value + if exp_value: + assert await port.get() == exp_value_converter(exp_value) @pytest.mark.parametrize( @@ -148,6 +146,14 @@ async def test_valid_port( "type": "blahblah", "displayOrder": 2.3, }, + { + "key": "some_file_with_file_in_value", + "label": "", + "description": "", + "type": "data:*/*", + "displayOrder": 2.3, + "value": __file__, + }, ], ) def test_invalid_port(port_cfg: Dict[str, Any]): From 06466d555740e522848a769a7fc19d94868008f2 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 17:13:40 +0100 Subject: [PATCH 22/74] correctly check value in port validation --- .../simcore-sdk/src/simcore_sdk/node_ports_v2/port.py | 11 +++++++++-- packages/simcore-sdk/tests/unit/test_port.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index e94978707bf4..9b245ab6e98f 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -31,10 +31,17 @@ class Port(ServiceProperty): _py_value_converter: Type[ItemConcreteValue] = PrivateAttr() _node_ports = PrivateAttr() - @validator("value", always=True) + @validator("value", pre=True, always=True) @classmethod def ensure_value(cls, v, values): - if v is None and values.get("default_value"): + if "property_type" in values and port_utils.is_file_type( + values["property_type"] + ): + if v is not None and not isinstance(v, (FileLink, DownloadLink, PortLink)): + raise ValueError( + f"[{values['property_type']}] must follow {FileLink.schema()}, {DownloadLink.schema()} or {PortLink.schema()}" + ) + elif v is None and values.get("default_value"): return values["default_value"] return v diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index b77b349bd6a7..d0d8baf50ea8 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -44,7 +44,7 @@ def camel_to_snake(name): }, (float), float, - -23.46, + -23.45, ), ( { From 5fbdad8bcca02551528d279fc24626d1da429bed Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 18:08:50 +0100 Subject: [PATCH 23/74] unit tests getting better, using fake storage calls --- .../services_api_mocks_for_aiohttp_clients.py | 31 ++++++++++++- .../src/simcore_sdk/node_ports_v2/port.py | 2 +- packages/simcore-sdk/tests/conftest.py | 1 + packages/simcore-sdk/tests/unit/test_port.py | 44 ++++++++++++++++++- 4 files changed, 74 insertions(+), 4 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py index a81de63f5736..a59ff2125116 100644 --- a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py +++ b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py @@ -1,12 +1,14 @@ import re +from unittest.mock import call import pytest from aioresponses import aioresponses from aioresponses.core import CallbackResult from models_library.projects_state import RunningState +from yarl import URL -def creation_cb(url, **kwargs): +def creation_cb(url, **kwargs) -> CallbackResult: assert "json" in kwargs, f"missing body in call to {url}" body = kwargs["json"] @@ -63,3 +65,30 @@ async def director_v2_subsystem_mock() -> aioresponses: mock.delete(delete_computation_pattern, status=204, repeat=True) yield mock + + +@pytest.fixture +async def storage_v0_subsystem_mock() -> aioresponses: + + """uses aioresponses to mock all calls of an aiohttpclient + WARNING: any request done through the client will go through aioresponses. It is + unfortunate but that means any valid request (like calling the test server) prefix must be set as passthrough. + Other than that it seems to behave nicely + """ + PASSTHROUGH_REQUESTS_PREFIXES = ["http://127.0.0.1", "ws://"] + + def get_download_link_cb(url: URL, **kwargs) -> CallbackResult: + file_id = url.path.rsplit("/files/")[1] + + return CallbackResult( + status=200, payload={"data": {"link": f"file://{file_id}"}} + ) + + get_download_link_pattern = re.compile( + r"^http://[a-z\-_]*storage:[0-9]+/v0/locations/[0-9]+/files/.+$" + ) + + with aioresponses(passthrough=PASSTHROUGH_REQUESTS_PREFIXES) as mock: + mock.get(get_download_link_pattern, callback=get_download_link_cb, repeat=True) + + yield mock diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 9b245ab6e98f..cec42c14b5b9 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -31,7 +31,7 @@ class Port(ServiceProperty): _py_value_converter: Type[ItemConcreteValue] = PrivateAttr() _node_ports = PrivateAttr() - @validator("value", pre=True, always=True) + @validator("value", always=True) @classmethod def ensure_value(cls, v, values): if "property_type" in values and port_utils.is_file_type( diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index 7b105bf72a08..3877097b3ab1 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -17,6 +17,7 @@ "pytest_simcore.postgres_service", "pytest_simcore.minio_service", "pytest_simcore.simcore_storage_service", + "pytest_simcore.services_api_mocks_for_aiohttp_clients", ] diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index d0d8baf50ea8..98af490dbd7a 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -4,21 +4,40 @@ # pylint:disable=no-member # pylint:disable=protected-access import re +from asyncio import Future from pathlib import Path from typing import Any, Dict, Type, Union import pytest from pydantic.error_wrappers import ValidationError +from simcore_sdk.node_ports import config +from simcore_sdk.node_ports_v2.links import FileLink from simcore_sdk.node_ports_v2.port import Port +@pytest.fixture +async def mock_download_file(mocker): + mock = mocker.patch( + "simcore_sdk.node_ports.filemanager.download_file_from_link", + return_value=Future(), + ) + mock.return_value.set_result(__file__) + yield mock + + +@pytest.fixture(autouse=True) +def node_ports_config(loop, storage_v0_subsystem_mock, mock_download_file): + config.USER_ID = "666" + config.STORAGE_ENDPOINT = "storage:8080" + + def camel_to_snake(name): name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() @pytest.mark.parametrize( - "port_cfg, exp_value_type, exp_value_converter, exp_value", + "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value", [ ( { @@ -32,6 +51,7 @@ def camel_to_snake(name): (int), int, 3, + 3, ), ( { @@ -45,6 +65,7 @@ def camel_to_snake(name): (float), float, -23.45, + -23.45, ), ( { @@ -58,6 +79,7 @@ def camel_to_snake(name): (bool), bool, True, + True, ), ( { @@ -72,6 +94,7 @@ def camel_to_snake(name): (bool), bool, False, + False, ), ( { @@ -84,6 +107,7 @@ def camel_to_snake(name): (Path, str), Path, None, + None, ), ( { @@ -97,6 +121,21 @@ def camel_to_snake(name): (Path, str), Path, None, + None, + ), + ( + { + "key": "some_file_with_file_in_defaulvalue", + "label": "", + "description": "", + "type": "data:*/*", + "displayOrder": 2.3, + "value": {"store": "0", "path": __file__}, + }, + (Path, str), + Path, + FileLink(store="0", path=__file__), + __file__, ), ], ) @@ -105,6 +144,7 @@ async def test_valid_port( exp_value_type: Type[Union[int, float, bool, str, Path]], exp_value_converter: Type[Union[int, float, bool, str, Path]], exp_value: Union[int, float, bool, str, Path], + exp_get_value: Union[int, float, bool, str, Path], ): port = Port(**port_cfg) @@ -119,7 +159,7 @@ async def test_valid_port( assert port.value == exp_value if exp_value: - assert await port.get() == exp_value_converter(exp_value) + assert await port.get() == exp_value_converter(exp_get_value) @pytest.mark.parametrize( From 206eb382baa393a7ec66f8c9e58c6103bc301dce Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 18:49:28 +0100 Subject: [PATCH 24/74] testing download links as well --- .../simcore_sdk/node_ports_v2/port_utils.py | 7 ++- packages/simcore-sdk/tests/unit/test_port.py | 44 ++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py index 8cbefa73f54a..bd7323dd63d0 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -80,14 +80,13 @@ async def pull_file_from_download_link( ) -> Path: log.debug( "Getting value from download link [%s] with label %s", - value["downloadLink"], - value.get("label", "undef"), + value.download_link, + value.label, ) - download_link = URL(value["downloadLink"]) local_path = data_items_utils.create_folder_path(key) downloaded_file = await filemanager.download_file_from_link( - download_link, + value.download_link, local_path, file_name=next(iter(fileToKeyMap)) if fileToKeyMap else None, ) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index 98af490dbd7a..4f891dda9f3e 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -11,7 +11,7 @@ import pytest from pydantic.error_wrappers import ValidationError from simcore_sdk.node_ports import config -from simcore_sdk.node_ports_v2.links import FileLink +from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink from simcore_sdk.node_ports_v2.port import Port @@ -137,6 +137,45 @@ def camel_to_snake(name): FileLink(store="0", path=__file__), __file__, ), + ( + { + "key": "some_file_with_file_in_defaulvalue", + "label": "", + "description": "", + "type": "data:*/*", + "displayOrder": 2.3, + "value": { + "store": "0", + "path": __file__, + "dataset": "some blahblah", + "label": "some blahblah", + }, + }, + (Path, str), + Path, + FileLink( + store="0", path=__file__, dataset="some blahblah", label="some blahblah" + ), + __file__, + ), + ( + { + "key": "some_file_with_file_in_defaulvalue", + "label": "", + "description": "", + "type": "data:*/*", + "displayOrder": 2.3, + "value": { + "downloadLink": "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md", + }, + }, + (Path, str), + Path, + DownloadLink( + downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" + ), + __file__, + ), ], ) async def test_valid_port( @@ -152,7 +191,8 @@ async def test_valid_port( camel_key = camel_to_snake(k) if k == "type": camel_key = "property_type" - assert v == getattr(port, camel_key) + if k != "value": + assert v == getattr(port, camel_key) assert port._py_value_type == exp_value_type assert port._py_value_converter == exp_value_converter From 7d023b6fd315c2c13758f466c92b7eceace16456 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 19:01:50 +0100 Subject: [PATCH 25/74] check portlink --- packages/simcore-sdk/tests/unit/test_port.py | 41 +++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index 4f891dda9f3e..d24e0990e1b3 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -11,7 +11,7 @@ import pytest from pydantic.error_wrappers import ValidationError from simcore_sdk.node_ports import config -from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink +from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink from simcore_sdk.node_ports_v2.port import Port @@ -125,7 +125,7 @@ def camel_to_snake(name): ), ( { - "key": "some_file_with_file_in_defaulvalue", + "key": "some_file_with_file_in_storage", "label": "", "description": "", "type": "data:*/*", @@ -139,13 +139,13 @@ def camel_to_snake(name): ), ( { - "key": "some_file_with_file_in_defaulvalue", + "key": "some_file_with_file_in_storage", "label": "", "description": "", "type": "data:*/*", "displayOrder": 2.3, "value": { - "store": "0", + "store": "1", "path": __file__, "dataset": "some blahblah", "label": "some blahblah", @@ -154,13 +154,13 @@ def camel_to_snake(name): (Path, str), Path, FileLink( - store="0", path=__file__, dataset="some blahblah", label="some blahblah" + store="1", path=__file__, dataset="some blahblah", label="some blahblah" ), __file__, ), ( { - "key": "some_file_with_file_in_defaulvalue", + "key": "some_file_with_file_as_download_link", "label": "", "description": "", "type": "data:*/*", @@ -176,6 +176,26 @@ def camel_to_snake(name): ), __file__, ), + ( + { + "key": "some_file_with_file_as_port_link", + "label": "", + "description": "", + "type": "data:*/*", + "displayOrder": 2.3, + "value": { + "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + "output": "the_output_of_that_node", + }, + }, + (Path, str), + Path, + PortLink( + nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + output="the_output_of_that_node", + ), + __file__, + ), ], ) async def test_valid_port( @@ -185,7 +205,16 @@ async def test_valid_port( exp_value: Union[int, float, bool, str, Path], exp_get_value: Union[int, float, bool, str, Path], ): + class FakeNodePorts: + async def get(self, key): + return exp_get_value + + async def _node_ports_creator_cb(self, node_uuid: str): + return FakeNodePorts() + + fake_node_ports = FakeNodePorts() port = Port(**port_cfg) + port._node_ports = fake_node_ports for k, v in port_cfg.items(): camel_key = camel_to_snake(k) From 6509f4dc1c1aaf23c8d7c5b09118f433cdb0be6c Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 19:08:33 +0100 Subject: [PATCH 26/74] improve verbosity --- packages/simcore-sdk/tests/unit/test_port.py | 127 ++++++++++--------- 1 file changed, 67 insertions(+), 60 deletions(-) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index d24e0990e1b3..c43062eae284 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -5,6 +5,7 @@ # pylint:disable=protected-access import re from asyncio import Future +from collections import namedtuple from pathlib import Path from typing import Any, Dict, Type, Union @@ -36,11 +37,17 @@ def camel_to_snake(name): return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() +PortParams = namedtuple( + "PortParams", + "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value", +) + + @pytest.mark.parametrize( "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value", [ - ( - { + PortParams( + port_cfg={ "key": "some_integer", "label": "some label", "description": "some description", @@ -48,13 +55,13 @@ def camel_to_snake(name): "displayOrder": 2.3, "defaultValue": 3, }, - (int), - int, - 3, - 3, + exp_value_type=(int), + exp_value_converter=int, + exp_value=3, + exp_get_value=3, ), - ( - { + PortParams( + port_cfg={ "key": "some_number", "label": "", "description": "", @@ -62,13 +69,13 @@ def camel_to_snake(name): "displayOrder": 2.3, "defaultValue": -23.45, }, - (float), - float, - -23.45, - -23.45, + exp_value_type=(float), + exp_value_converter=float, + exp_value=-23.45, + exp_get_value=-23.45, ), - ( - { + PortParams( + port_cfg={ "key": "some_boolean", "label": "", "description": "", @@ -76,13 +83,13 @@ def camel_to_snake(name): "displayOrder": 2.3, "defaultValue": True, }, - (bool), - bool, - True, - True, + exp_value_type=(bool), + exp_value_converter=bool, + exp_value=True, + exp_get_value=True, ), - ( - { + PortParams( + port_cfg={ "key": "some_boolean", "label": "", "description": "", @@ -91,26 +98,26 @@ def camel_to_snake(name): "defaultValue": True, "value": False, }, - (bool), - bool, - False, - False, + exp_value_type=(bool), + exp_value_converter=bool, + exp_value=False, + exp_get_value=False, ), - ( - { + PortParams( + port_cfg={ "key": "some_file", "label": "", "description": "", "type": "data:*/*", "displayOrder": 2.3, }, - (Path, str), - Path, - None, - None, + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=None, + exp_get_value=None, ), - ( - { + PortParams( + port_cfg={ "key": "some_file_with_file_in_defaulvalue", "label": "", "description": "", @@ -118,13 +125,13 @@ def camel_to_snake(name): "displayOrder": 2.3, "defaultValue": __file__, }, - (Path, str), - Path, - None, - None, + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=None, + exp_get_value=None, ), - ( - { + PortParams( + port_cfg={ "key": "some_file_with_file_in_storage", "label": "", "description": "", @@ -132,13 +139,13 @@ def camel_to_snake(name): "displayOrder": 2.3, "value": {"store": "0", "path": __file__}, }, - (Path, str), - Path, - FileLink(store="0", path=__file__), - __file__, + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=FileLink(store="0", path=__file__), + exp_get_value=__file__, ), - ( - { + PortParams( + port_cfg={ "key": "some_file_with_file_in_storage", "label": "", "description": "", @@ -151,15 +158,15 @@ def camel_to_snake(name): "label": "some blahblah", }, }, - (Path, str), - Path, - FileLink( + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=FileLink( store="1", path=__file__, dataset="some blahblah", label="some blahblah" ), - __file__, + exp_get_value=__file__, ), - ( - { + PortParams( + port_cfg={ "key": "some_file_with_file_as_download_link", "label": "", "description": "", @@ -169,15 +176,15 @@ def camel_to_snake(name): "downloadLink": "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md", }, }, - (Path, str), - Path, - DownloadLink( + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=DownloadLink( downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" ), - __file__, + exp_get_value=__file__, ), - ( - { + PortParams( + port_cfg={ "key": "some_file_with_file_as_port_link", "label": "", "description": "", @@ -188,13 +195,13 @@ def camel_to_snake(name): "output": "the_output_of_that_node", }, }, - (Path, str), - Path, - PortLink( + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=PortLink( nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", output="the_output_of_that_node", ), - __file__, + exp_get_value=__file__, ), ], ) From d37770252635c804cf545430b9bfc9c7accf5ab1 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 19:09:38 +0100 Subject: [PATCH 27/74] linter --- packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py | 1 + .../simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index cec42c14b5b9..e351641dcebe 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -74,6 +74,7 @@ async def get(self) -> ItemConcreteValue: if isinstance(self.value, PortLink): # this is a link to another node value = await port_utils.get_value_from_link( + # pylint: disable=protected-access self.key, self.value, self.file_to_key_map, diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py index bd7323dd63d0..36caea4fb583 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -3,8 +3,6 @@ from pathlib import Path from typing import Any, Callable, Coroutine, Dict -from yarl import URL - from ..node_ports import config, data_items_utils, filemanager from .links import DownloadLink, FileLink, ItemConcreteValue, PortLink From 5ce66e4a73c8d96d368e94d1e0e754bfe0aba4a9 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 19:25:56 +0100 Subject: [PATCH 28/74] getting there with unit testing port --- .../services_api_mocks_for_aiohttp_clients.py | 10 +++- .../src/simcore_sdk/node_ports_v2/port.py | 7 +-- packages/simcore-sdk/tests/unit/test_port.py | 47 +++++++++++++++++-- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py index a59ff2125116..d6a3e1bc06c5 100644 --- a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py +++ b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py @@ -88,7 +88,15 @@ def get_download_link_cb(url: URL, **kwargs) -> CallbackResult: r"^http://[a-z\-_]*storage:[0-9]+/v0/locations/[0-9]+/files/.+$" ) + get_locations_link_pattern = re.compile( + r"^http://[a-z\-_]*storage:[0-9]+/v0/locations.*$" + ) + with aioresponses(passthrough=PASSTHROUGH_REQUESTS_PREFIXES) as mock: mock.get(get_download_link_pattern, callback=get_download_link_cb, repeat=True) - + mock.get( + get_locations_link_pattern, + status=200, + payload={"data": [{"name": "simcore.s3", "id": "0"}]}, + ) yield mock diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index e351641dcebe..b7add0a00c2d 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -100,11 +100,12 @@ async def set(self, new_value: ItemConcreteValue): log.debug( "setting %s[%s] with value %s", self.key, self.property_type, new_value ) - if not isinstance(new_value, self._py_value_type): - raise InvalidItemTypeError(self.property_type, new_value) - # convert the concrete value to a data value data_value = self._py_value_converter(new_value) + + if not isinstance(data_value, self._py_value_type): + raise InvalidItemTypeError(self.property_type, new_value) + if port_utils.is_file_type(self.property_type): if not data_value.exists() or not data_value.is_file(): raise InvalidItemTypeError(self.property_type, new_value) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index c43062eae284..fd86112fc7ef 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -39,12 +39,12 @@ def camel_to_snake(name): PortParams = namedtuple( "PortParams", - "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value", + "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", ) @pytest.mark.parametrize( - "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value", + "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", [ PortParams( port_cfg={ @@ -59,6 +59,9 @@ def camel_to_snake(name): exp_value_converter=int, exp_value=3, exp_get_value=3, + new_value=7, + exp_new_value=7, + exp_new_get_value=7, ), PortParams( port_cfg={ @@ -73,6 +76,9 @@ def camel_to_snake(name): exp_value_converter=float, exp_value=-23.45, exp_get_value=-23.45, + new_value=7, + exp_new_value=7.0, + exp_new_get_value=7.0, ), PortParams( port_cfg={ @@ -87,6 +93,9 @@ def camel_to_snake(name): exp_value_converter=bool, exp_value=True, exp_get_value=True, + new_value=False, + exp_new_value=False, + exp_new_get_value=False, ), PortParams( port_cfg={ @@ -102,6 +111,9 @@ def camel_to_snake(name): exp_value_converter=bool, exp_value=False, exp_get_value=False, + new_value=True, + exp_new_value=True, + exp_new_get_value=True, ), PortParams( port_cfg={ @@ -115,6 +127,9 @@ def camel_to_snake(name): exp_value_converter=Path, exp_value=None, exp_get_value=None, + new_value=__file__, + exp_new_value=FileLink(store="0", path=__file__), + exp_new_get_value=__file__, ), PortParams( port_cfg={ @@ -129,6 +144,9 @@ def camel_to_snake(name): exp_value_converter=Path, exp_value=None, exp_get_value=None, + new_value=__file__, + exp_new_value=FileLink(store="0", path=__file__), + exp_new_get_value=__file__, ), PortParams( port_cfg={ @@ -143,6 +161,9 @@ def camel_to_snake(name): exp_value_converter=Path, exp_value=FileLink(store="0", path=__file__), exp_get_value=__file__, + new_value=None, + exp_new_value=None, + exp_new_get_value=None, ), PortParams( port_cfg={ @@ -164,6 +185,9 @@ def camel_to_snake(name): store="1", path=__file__, dataset="some blahblah", label="some blahblah" ), exp_get_value=__file__, + new_value=__file__, + exp_new_value=FileLink(store="0", path=__file__), + exp_new_get_value=__file__, ), PortParams( port_cfg={ @@ -182,6 +206,9 @@ def camel_to_snake(name): downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" ), exp_get_value=__file__, + new_value=__file__, + exp_new_value=FileLink(store="0", path=__file__), + exp_new_get_value=__file__, ), PortParams( port_cfg={ @@ -202,6 +229,9 @@ def camel_to_snake(name): output="the_output_of_that_node", ), exp_get_value=__file__, + new_value=__file__, + exp_new_value=FileLink(store="0", path=__file__), + exp_new_get_value=__file__, ), ], ) @@ -209,8 +239,11 @@ async def test_valid_port( port_cfg: Dict[str, Any], exp_value_type: Type[Union[int, float, bool, str, Path]], exp_value_converter: Type[Union[int, float, bool, str, Path]], - exp_value: Union[int, float, bool, str, Path], + exp_value: Union[int, float, bool, str, Path, FileLink, DownloadLink, PortLink], exp_get_value: Union[int, float, bool, str, Path], + new_value: Union[int, float, bool, str, Path], + exp_new_value: Union[int, float, bool, str, Path, FileLink], + exp_new_get_value: Union[int, float, bool, str, Path], ): class FakeNodePorts: async def get(self, key): @@ -219,10 +252,14 @@ async def get(self, key): async def _node_ports_creator_cb(self, node_uuid: str): return FakeNodePorts() + async def save_to_db_cb(self, node_ports): + return + fake_node_ports = FakeNodePorts() port = Port(**port_cfg) port._node_ports = fake_node_ports + # check schema for k, v in port_cfg.items(): camel_key = camel_to_snake(k) if k == "type": @@ -230,6 +267,7 @@ async def _node_ports_creator_cb(self, node_uuid: str): if k != "value": assert v == getattr(port, camel_key) + # check payload assert port._py_value_type == exp_value_type assert port._py_value_converter == exp_value_converter @@ -237,6 +275,9 @@ async def _node_ports_creator_cb(self, node_uuid: str): if exp_value: assert await port.get() == exp_value_converter(exp_get_value) + # set a new value + await port.set(new_value) + @pytest.mark.parametrize( "port_cfg", From e4c14945b19e8884580a7bc5ae63f00f51b1fa33 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 19:27:34 +0100 Subject: [PATCH 29/74] last check before running out of the train --- packages/simcore-sdk/tests/unit/test_port.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index fd86112fc7ef..83ca92e84ecc 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -26,8 +26,20 @@ async def mock_download_file(mocker): yield mock +@pytest.fixture +async def mock_upload_file(mocker): + mock = mocker.patch( + "simcore_sdk.node_ports.filemanager.upload_file", + return_value=Future(), + ) + mock.return_value.set_result("") + yield mock + + @pytest.fixture(autouse=True) -def node_ports_config(loop, storage_v0_subsystem_mock, mock_download_file): +def node_ports_config( + loop, storage_v0_subsystem_mock, mock_download_file, mock_upload_file +): config.USER_ID = "666" config.STORAGE_ENDPOINT = "storage:8080" From 0b40a7d1a34bb72ab527c63ce49ab8484a483e0d Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 22:46:39 +0100 Subject: [PATCH 30/74] Port fully tested --- .../src/simcore_sdk/node_ports/exceptions.py | 17 +- .../src/simcore_sdk/node_ports_v2/port.py | 23 +- packages/simcore-sdk/tests/unit/test_port.py | 223 +++++++++++------- 3 files changed, 155 insertions(+), 108 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py index f5e15714b012..230fa3f015c3 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py @@ -1,10 +1,12 @@ +from typing import Optional + """Defines the different exceptions that may arise in the nodeports package""" class NodeportsException(Exception): """Basic exception for errors raised in nodeports""" - def __init__(self, msg=None): + def __init__(self, msg: Optional[str] = None): super().__init__(msg or "An error occured in simcore") @@ -30,7 +32,7 @@ def __init__(self, expected_version, found_version): class UnboundPortError(NodeportsException, IndexError): """Accessed port is not configured""" - def __init__(self, port_index, msg=None): + def __init__(self, port_index, msg: Optional[str] = None): super().__init__(f"No port bound at index {port_index}") self.port_index = port_index @@ -38,7 +40,7 @@ def __init__(self, port_index, msg=None): class InvalidKeyError(NodeportsException): """Accessed key does not exist""" - def __init__(self, item_key, msg=None): + def __init__(self, item_key: str, msg: Optional[str] = None): super().__init__(f"No port bound with key {item_key}") self.item_key = item_key @@ -46,9 +48,10 @@ def __init__(self, item_key, msg=None): class InvalidItemTypeError(NodeportsException): """Item type incorrect""" - def __init__(self, item_type, item_value): + def __init__(self, item_type: str, item_value: str, msg: Optional[str] = None): super().__init__( - f"Invalid item type, value [{item_value}] does not qualify as type [{item_type}]" + msg + or f"Invalid item type, value [{item_value}] does not qualify as type [{item_type}]" ) self.item_type = item_type self.item_value = item_value @@ -57,7 +60,7 @@ def __init__(self, item_type, item_value): class InvalidProtocolError(NodeportsException): """Invalid protocol used""" - def __init__(self, dct, msg=None): + def __init__(self, dct, msg: Optional[str] = None): super().__init__(f"Invalid protocol used: {dct}\n{msg}") self.dct = dct @@ -73,7 +76,7 @@ class StorageServerIssue(NodeportsException): class S3TransferError(NodeportsException): """S3 transfer error""" - def __init__(self, msg=None): + def __init__(self, msg: Optional[str] = None): super().__init__(msg or "Error while transferring to/from S3 storage") diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index b7add0a00c2d..148e7384a676 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -100,18 +100,19 @@ async def set(self, new_value: ItemConcreteValue): log.debug( "setting %s[%s] with value %s", self.key, self.property_type, new_value ) - # convert the concrete value to a data value - data_value = self._py_value_converter(new_value) - - if not isinstance(data_value, self._py_value_type): - raise InvalidItemTypeError(self.property_type, new_value) - - if port_utils.is_file_type(self.property_type): - if not data_value.exists() or not data_value.is_file(): - raise InvalidItemTypeError(self.property_type, new_value) - data_value: FileLink = await port_utils.push_file_to_store(data_value) + converted_value = None + if new_value is not None: + # convert the concrete value to a data value + converted_value = self._py_value_converter(new_value) + + if port_utils.is_file_type(self.property_type): + if not converted_value.exists() or not converted_value.is_file(): + raise InvalidItemTypeError(self.property_type, new_value) + converted_value: FileLink = await port_utils.push_file_to_store( + converted_value + ) - self.value = data_value + self.value = converted_value await self._node_ports.save_to_db_cb(self._node_ports) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index 83ca92e84ecc..99a31ef4d275 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -7,15 +7,32 @@ from asyncio import Future from collections import namedtuple from pathlib import Path +from random import randint from typing import Any, Dict, Type, Union import pytest from pydantic.error_wrappers import ValidationError from simcore_sdk.node_ports import config +from simcore_sdk.node_ports.exceptions import InvalidItemTypeError from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink from simcore_sdk.node_ports_v2.port import Port +@pytest.fixture(scope="module") +def project_id() -> str: + return "cd0d8dbb-3263-44dc-921c-49c075ac0dd9" + + +@pytest.fixture(scope="module") +def node_uuid() -> str: + return "609b7af4-6861-4aa7-a16e-730ea8125190" + + +@pytest.fixture(scope="module") +def user_id() -> str: + return str(randint(1, 666)) + + @pytest.fixture async def mock_download_file(mocker): mock = mocker.patch( @@ -32,15 +49,23 @@ async def mock_upload_file(mocker): "simcore_sdk.node_ports.filemanager.upload_file", return_value=Future(), ) - mock.return_value.set_result("") + mock.return_value.set_result("0") yield mock @pytest.fixture(autouse=True) def node_ports_config( - loop, storage_v0_subsystem_mock, mock_download_file, mock_upload_file + loop, + storage_v0_subsystem_mock, + mock_download_file, + mock_upload_file, + project_id: str, + user_id: str, + node_uuid: str, ): - config.USER_ID = "666" + config.USER_ID = user_id + config.PROJECT_ID = project_id + config.NODE_UUID = node_uuid config.STORAGE_ENDPOINT = "storage:8080" @@ -55,18 +80,23 @@ def camel_to_snake(name): ) +def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: + valid_config = { + "key": f"some_{conf_type}", + "label": "some label", + "description": "some description", + "type": conf_type, + "displayOrder": 2.3, + } + valid_config.update(kwargs) + return valid_config + + @pytest.mark.parametrize( "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", [ PortParams( - port_cfg={ - "key": "some_integer", - "label": "some label", - "description": "some description", - "type": "integer", - "displayOrder": 2.3, - "defaultValue": 3, - }, + port_cfg=create_valid_port_config("integer", defaultValue=3), exp_value_type=(int), exp_value_converter=int, exp_value=3, @@ -76,14 +106,7 @@ def camel_to_snake(name): exp_new_get_value=7, ), PortParams( - port_cfg={ - "key": "some_number", - "label": "", - "description": "", - "type": "number", - "displayOrder": 2.3, - "defaultValue": -23.45, - }, + port_cfg=create_valid_port_config("number", defaultValue=-23.45), exp_value_type=(float), exp_value_converter=float, exp_value=-23.45, @@ -93,14 +116,7 @@ def camel_to_snake(name): exp_new_get_value=7.0, ), PortParams( - port_cfg={ - "key": "some_boolean", - "label": "", - "description": "", - "type": "boolean", - "displayOrder": 2.3, - "defaultValue": True, - }, + port_cfg=create_valid_port_config("boolean", defaultValue=True), exp_value_type=(bool), exp_value_converter=bool, exp_value=True, @@ -110,15 +126,9 @@ def camel_to_snake(name): exp_new_get_value=False, ), PortParams( - port_cfg={ - "key": "some_boolean", - "label": "", - "description": "", - "type": "boolean", - "displayOrder": 2.3, - "defaultValue": True, - "value": False, - }, + port_cfg=create_valid_port_config( + "boolean", defaultValue=True, value=False + ), exp_value_type=(bool), exp_value_converter=bool, exp_value=False, @@ -128,47 +138,39 @@ def camel_to_snake(name): exp_new_get_value=True, ), PortParams( - port_cfg={ - "key": "some_file", - "label": "", - "description": "", - "type": "data:*/*", - "displayOrder": 2.3, - }, + port_cfg=create_valid_port_config("data:*/*", key="no_file"), exp_value_type=(Path, str), exp_value_converter=Path, exp_value=None, exp_get_value=None, new_value=__file__, - exp_new_value=FileLink(store="0", path=__file__), + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", + ), exp_new_get_value=__file__, ), PortParams( - port_cfg={ - "key": "some_file_with_file_in_defaulvalue", - "label": "", - "description": "", - "type": "data:*/*", - "displayOrder": 2.3, - "defaultValue": __file__, - }, + port_cfg=create_valid_port_config( + "data:*/*", key="no_file_with_default", defaultValue=__file__ + ), exp_value_type=(Path, str), exp_value_converter=Path, exp_value=None, exp_get_value=None, new_value=__file__, - exp_new_value=FileLink(store="0", path=__file__), + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", + ), exp_new_get_value=__file__, ), PortParams( - port_cfg={ - "key": "some_file_with_file_in_storage", - "label": "", - "description": "", - "type": "data:*/*", - "displayOrder": 2.3, - "value": {"store": "0", "path": __file__}, - }, + port_cfg=create_valid_port_config( + "data:*/*", + key="some_file", + value={"store": "0", "path": __file__}, + ), exp_value_type=(Path, str), exp_value_converter=Path, exp_value=FileLink(store="0", path=__file__), @@ -178,19 +180,16 @@ def camel_to_snake(name): exp_new_get_value=None, ), PortParams( - port_cfg={ - "key": "some_file_with_file_in_storage", - "label": "", - "description": "", - "type": "data:*/*", - "displayOrder": 2.3, - "value": { + port_cfg=create_valid_port_config( + "data:*/*", + key="some_file_on_datcore", + value={ "store": "1", "path": __file__, "dataset": "some blahblah", "label": "some blahblah", }, - }, + ), exp_value_type=(Path, str), exp_value_converter=Path, exp_value=FileLink( @@ -198,20 +197,20 @@ def camel_to_snake(name): ), exp_get_value=__file__, new_value=__file__, - exp_new_value=FileLink(store="0", path=__file__), + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", + ), exp_new_get_value=__file__, ), PortParams( - port_cfg={ - "key": "some_file_with_file_as_download_link", - "label": "", - "description": "", - "type": "data:*/*", - "displayOrder": 2.3, - "value": { - "downloadLink": "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md", + port_cfg=create_valid_port_config( + "data:*/*", + key="download_link", + value={ + "downloadLink": "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" }, - }, + ), exp_value_type=(Path, str), exp_value_converter=Path, exp_value=DownloadLink( @@ -219,21 +218,21 @@ def camel_to_snake(name): ), exp_get_value=__file__, new_value=__file__, - exp_new_value=FileLink(store="0", path=__file__), + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", + ), exp_new_get_value=__file__, ), PortParams( - port_cfg={ - "key": "some_file_with_file_as_port_link", - "label": "", - "description": "", - "type": "data:*/*", - "displayOrder": 2.3, - "value": { + port_cfg=create_valid_port_config( + "data:*/*", + key="file_port_link", + value={ "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", "output": "the_output_of_that_node", }, - }, + ), exp_value_type=(Path, str), exp_value_converter=Path, exp_value=PortLink( @@ -242,9 +241,32 @@ def camel_to_snake(name): ), exp_get_value=__file__, new_value=__file__, - exp_new_value=FileLink(store="0", path=__file__), + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", + ), exp_new_get_value=__file__, ), + PortParams( + port_cfg=create_valid_port_config( + "number", + key="number_port_link", + value={ + "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + "output": "the_output_of_that_node", + }, + ), + exp_value_type=(float), + exp_value_converter=float, + exp_value=PortLink( + nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + output="the_output_of_that_node", + ), + exp_get_value=562.45, + new_value=None, + exp_new_value=None, + exp_new_get_value=None, + ), ], ) async def test_valid_port( @@ -284,11 +306,18 @@ async def save_to_db_cb(self, node_ports): assert port._py_value_converter == exp_value_converter assert port.value == exp_value - if exp_value: + if exp_get_value is None: + assert await port.get() == None + else: assert await port.get() == exp_value_converter(exp_get_value) # set a new value await port.set(new_value) + assert port.value == exp_new_value + if exp_new_get_value is None: + assert await port.get() == None + else: + assert await port.get() == exp_value_converter(exp_new_get_value) @pytest.mark.parametrize( @@ -328,3 +357,17 @@ async def save_to_db_cb(self, node_ports): def test_invalid_port(port_cfg: Dict[str, Any]): with pytest.raises(ValidationError): Port(**port_cfg) + + +@pytest.mark.parametrize( + "port_cfg", [(create_valid_port_config("data:*/*", key="set_some_inexisting_file"))] +) +async def test_invalid_file_type_setter(port_cfg: Dict[str, Any]): + port = Port(**port_cfg) + # set a file that does not exist + with pytest.raises(InvalidItemTypeError): + await port.set("some/dummy/file/name") + + # set a folder fails too + with pytest.raises(InvalidItemTypeError): + await port.set(Path(__file__).parent) From 5bc9747fe73c0456a23d02a1fa75556d09972332 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 4 Dec 2020 22:50:50 +0100 Subject: [PATCH 31/74] create new file --- .../simcore_sdk/node_ports_v2/nodeports_v2.py | 3 +- .../src/simcore_sdk/node_ports_v2/port.py | 38 ++----------------- .../node_ports_v2/ports_mapping.py | 38 +++++++++++++++++++ 3 files changed, 43 insertions(+), 36 deletions(-) create mode 100644 packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 82b805cefafa..11eb971c6d77 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -6,8 +6,9 @@ from ..node_ports.dbmanager import DBManager from ..node_ports.exceptions import PortNotFound, UnboundPortError -from .port import InputsList, ItemConcreteValue, OutputsList +from .port import ItemConcreteValue from .port_utils import is_file_type +from .ports_mapping import InputsList, OutputsList log = logging.getLogger(__name__) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 148e7384a676..d8f0c7afb346 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -1,12 +1,12 @@ import logging from pathlib import Path from pprint import pformat -from typing import Dict, Optional, Tuple, Type, Union +from typing import Dict, Optional, Tuple, Type from models_library.services import PROPERTY_KEY_RE, ServiceProperty -from pydantic import BaseModel, Field, PrivateAttr, validator +from pydantic import Field, PrivateAttr, validator -from ..node_ports.exceptions import InvalidItemTypeError, UnboundPortError +from ..node_ports.exceptions import InvalidItemTypeError from . import port_utils from .links import DataItemValue, DownloadLink, FileLink, ItemConcreteValue, PortLink @@ -114,35 +114,3 @@ async def set(self, new_value: ItemConcreteValue): self.value = converted_value await self._node_ports.save_to_db_cb(self._node_ports) - - -PortKey = str - - -class PortsMapping(BaseModel): - __root__: Dict[PortKey, Port] - - def __getitem__(self, key: Union[int, PortKey]) -> Port: - if isinstance(key, int): - if key < len(self.__root__): - key = list(self.__root__.keys())[key] - if not key in self.__root__: - raise UnboundPortError(key) - return self.__root__[key] - - def __iter__(self): - return iter(self.__root__) - - def items(self): - return self.__root__.items() - - def values(self): - return self.__root__.values() - - -class InputsList(PortsMapping): - __root__: Dict[PortKey, Port] - - -class OutputsList(PortsMapping): - __root__: Dict[PortKey, Port] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py new file mode 100644 index 000000000000..1349fed49d06 --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py @@ -0,0 +1,38 @@ +from typing import Dict, Union + +from models_library.services import PROPERTY_KEY_RE +from pydantic import BaseModel, constr + +from ..node_ports.exceptions import UnboundPortError +from .port import Port + +PortKey = constr(regex=PROPERTY_KEY_RE) + + +class PortsMapping(BaseModel): + __root__: Dict[PortKey, Port] + + def __getitem__(self, key: Union[int, PortKey]) -> Port: + if isinstance(key, int): + if key < len(self.__root__): + key = list(self.__root__.keys())[key] + if not key in self.__root__: + raise UnboundPortError(key) + return self.__root__[key] + + def __iter__(self): + return iter(self.__root__) + + def items(self): + return self.__root__.items() + + def values(self): + return self.__root__.values() + + +class InputsList(PortsMapping): + __root__: Dict[PortKey, Port] + + +class OutputsList(PortsMapping): + __root__: Dict[PortKey, Port] From c3d06c3cbdf4eda01779aecab20bef5d3d1d5724 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sat, 5 Dec 2020 23:32:27 +0100 Subject: [PATCH 32/74] fixed mocking of download data --- packages/simcore-sdk/tests/unit/test_port.py | 443 ++++++++++++------- 1 file changed, 280 insertions(+), 163 deletions(-) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index 99a31ef4d275..37b4fa471787 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -4,18 +4,22 @@ # pylint:disable=no-member # pylint:disable=protected-access import re +import shutil +import tempfile from asyncio import Future from collections import namedtuple from pathlib import Path from random import randint -from typing import Any, Dict, Type, Union +from typing import Any, Dict, Optional, Type, Union import pytest +from aiohttp.client import ClientSession from pydantic.error_wrappers import ValidationError from simcore_sdk.node_ports import config from simcore_sdk.node_ports.exceptions import InvalidItemTypeError from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink from simcore_sdk.node_ports_v2.port import Port +from yarl import URL @pytest.fixture(scope="module") @@ -33,14 +37,64 @@ def user_id() -> str: return str(randint(1, 666)) +THIS_NODE_FILE_NAME: str = f"{tempfile.gettempdir()}/this_node_file.txt" +DOWNLOAD_FILE_DIR: Path = Path(tempfile.gettempdir(), "simcorefiles") +ANOTHER_NODE_FILE_NAME: Path = Path(tempfile.gettempdir(), "another_node_file.txt") + + @pytest.fixture -async def mock_download_file(mocker): - mock = mocker.patch( - "simcore_sdk.node_ports.filemanager.download_file_from_link", - return_value=Future(), +def this_node_file() -> Path: + file_path = Path(THIS_NODE_FILE_NAME) + file_path.write_text("some dummy data") + assert file_path.exists() + yield file_path + if file_path.exists(): + file_path.unlink() + + +@pytest.fixture +def another_node_file() -> Path: + file_path = Path(tempfile.gettempdir(), "another_node_file.txt") + file_path.write_text("some dummy data") + assert file_path.exists() + yield file_path + if file_path.exists(): + file_path.unlink() + + +@pytest.fixture +def downloaded_file_folder() -> Path: + destination_path = DOWNLOAD_FILE_DIR + yield destination_path + if destination_path.exists(): + shutil.rmtree(destination_path) + + +@pytest.fixture +async def mock_download_file( + monkeypatch, + this_node_file: Path, + project_id: str, + node_uuid: str, + downloaded_file_folder: Path, +): + async def mock_download_file_from_link( + download_link: URL, + local_folder: Path, + session: Optional[ClientSession] = None, + file_name: Optional[str] = None, + ) -> Path: + assert str(local_folder).startswith(str(DOWNLOAD_FILE_DIR)) + destination_path = local_folder / this_node_file.name + destination_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(this_node_file, destination_path) + return destination_path + + from simcore_sdk.node_ports import filemanager + + monkeypatch.setattr( + filemanager, "download_file_from_link", mock_download_file_from_link ) - mock.return_value.set_result(__file__) - yield mock @pytest.fixture @@ -95,177 +149,233 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: @pytest.mark.parametrize( "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", [ - PortParams( - port_cfg=create_valid_port_config("integer", defaultValue=3), - exp_value_type=(int), - exp_value_converter=int, - exp_value=3, - exp_get_value=3, - new_value=7, - exp_new_value=7, - exp_new_get_value=7, - ), - PortParams( - port_cfg=create_valid_port_config("number", defaultValue=-23.45), - exp_value_type=(float), - exp_value_converter=float, - exp_value=-23.45, - exp_get_value=-23.45, - new_value=7, - exp_new_value=7.0, - exp_new_get_value=7.0, - ), - PortParams( - port_cfg=create_valid_port_config("boolean", defaultValue=True), - exp_value_type=(bool), - exp_value_converter=bool, - exp_value=True, - exp_get_value=True, - new_value=False, - exp_new_value=False, - exp_new_get_value=False, - ), - PortParams( - port_cfg=create_valid_port_config( - "boolean", defaultValue=True, value=False + pytest.param( + *PortParams( + port_cfg=create_valid_port_config("integer", defaultValue=3), + exp_value_type=(int), + exp_value_converter=int, + exp_value=3, + exp_get_value=3, + new_value=7, + exp_new_value=7, + exp_new_get_value=7, ), - exp_value_type=(bool), - exp_value_converter=bool, - exp_value=False, - exp_get_value=False, - new_value=True, - exp_new_value=True, - exp_new_get_value=True, + id="integer value with default", ), - PortParams( - port_cfg=create_valid_port_config("data:*/*", key="no_file"), - exp_value_type=(Path, str), - exp_value_converter=Path, - exp_value=None, - exp_get_value=None, - new_value=__file__, - exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", + pytest.param( + *PortParams( + port_cfg=create_valid_port_config("number", defaultValue=-23.45), + exp_value_type=(float), + exp_value_converter=float, + exp_value=-23.45, + exp_get_value=-23.45, + new_value=7, + exp_new_value=7.0, + exp_new_get_value=7.0, ), - exp_new_get_value=__file__, + id="number value with default", ), - PortParams( - port_cfg=create_valid_port_config( - "data:*/*", key="no_file_with_default", defaultValue=__file__ - ), - exp_value_type=(Path, str), - exp_value_converter=Path, - exp_value=None, - exp_get_value=None, - new_value=__file__, - exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", + pytest.param( + *PortParams( + port_cfg=create_valid_port_config("boolean", defaultValue=True), + exp_value_type=(bool), + exp_value_converter=bool, + exp_value=True, + exp_get_value=True, + new_value=False, + exp_new_value=False, + exp_new_get_value=False, ), - exp_new_get_value=__file__, + id="boolean value with default", ), - PortParams( - port_cfg=create_valid_port_config( - "data:*/*", - key="some_file", - value={"store": "0", "path": __file__}, + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "boolean", defaultValue=True, value=False + ), + exp_value_type=(bool), + exp_value_converter=bool, + exp_value=False, + exp_get_value=False, + new_value=True, + exp_new_value=True, + exp_new_get_value=True, ), - exp_value_type=(Path, str), - exp_value_converter=Path, - exp_value=FileLink(store="0", path=__file__), - exp_get_value=__file__, - new_value=None, - exp_new_value=None, - exp_new_get_value=None, + id="boolean value with default and value", ), - PortParams( - port_cfg=create_valid_port_config( - "data:*/*", - key="some_file_on_datcore", - value={ - "store": "1", - "path": __file__, - "dataset": "some blahblah", - "label": "some blahblah", - }, + pytest.param( + *PortParams( + port_cfg=create_valid_port_config("data:*/*", key="no_file"), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=None, + exp_get_value=None, + new_value=THIS_NODE_FILE_NAME, + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + ), + exp_new_get_value=DOWNLOAD_FILE_DIR + / "no_file" + / Path(THIS_NODE_FILE_NAME).name, ), - exp_value_type=(Path, str), - exp_value_converter=Path, - exp_value=FileLink( - store="1", path=__file__, dataset="some blahblah", label="some blahblah" - ), - exp_get_value=__file__, - new_value=__file__, - exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", - ), - exp_new_get_value=__file__, + id="file type with no payload", ), - PortParams( - port_cfg=create_valid_port_config( - "data:*/*", - key="download_link", - value={ - "downloadLink": "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" - }, - ), - exp_value_type=(Path, str), - exp_value_converter=Path, - exp_value=DownloadLink( - downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" - ), - exp_get_value=__file__, - new_value=__file__, - exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="no_file_with_default", + defaultValue=THIS_NODE_FILE_NAME, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=None, + exp_get_value=None, + new_value=THIS_NODE_FILE_NAME, + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + ), + exp_new_get_value=DOWNLOAD_FILE_DIR + / "no_file_with_default" + / Path(THIS_NODE_FILE_NAME).name, ), - exp_new_get_value=__file__, + id="file link with no payload and default value", ), - PortParams( - port_cfg=create_valid_port_config( - "data:*/*", - key="file_port_link", - value={ - "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", - "output": "the_output_of_that_node", - }, + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="some_file", + value={"store": "0", "path": THIS_NODE_FILE_NAME}, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=FileLink(store="0", path=THIS_NODE_FILE_NAME), + exp_get_value=DOWNLOAD_FILE_DIR + / "some_file" + / Path(THIS_NODE_FILE_NAME).name, + new_value=None, + exp_new_value=None, + exp_new_get_value=None, ), - exp_value_type=(Path, str), - exp_value_converter=Path, - exp_value=PortLink( - nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", - output="the_output_of_that_node", + id="file link with payload that get reset", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="some_file_on_datcore", + value={ + "store": "1", + "path": THIS_NODE_FILE_NAME, + "dataset": "some blahblah", + "label": "some blahblah", + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=FileLink( + store="1", + path=THIS_NODE_FILE_NAME, + dataset="some blahblah", + label="some blahblah", + ), + exp_get_value=DOWNLOAD_FILE_DIR + / "some_file_on_datcore" + / Path(THIS_NODE_FILE_NAME).name, + new_value=THIS_NODE_FILE_NAME, + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + ), + exp_new_get_value=DOWNLOAD_FILE_DIR + / "some_file_on_datcore" + / Path(THIS_NODE_FILE_NAME).name, ), - exp_get_value=__file__, - new_value=__file__, - exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(__file__).name}", + id="file link with payload on store 1", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="download_link", + value={ + "downloadLink": "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=DownloadLink( + downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" + ), + exp_get_value=DOWNLOAD_FILE_DIR + / "download_link" + / Path(THIS_NODE_FILE_NAME).name, + new_value=THIS_NODE_FILE_NAME, + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + ), + exp_new_get_value=DOWNLOAD_FILE_DIR + / "download_link" + / Path(THIS_NODE_FILE_NAME).name, ), - exp_new_get_value=__file__, + id="download link file type gets set back on store", ), - PortParams( - port_cfg=create_valid_port_config( - "number", - key="number_port_link", - value={ - "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", - "output": "the_output_of_that_node", - }, + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="file_port_link", + value={ + "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + "output": "the_output_of_that_node", + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=PortLink( + nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + output="the_output_of_that_node", + ), + exp_get_value=DOWNLOAD_FILE_DIR + / "file_port_link" + / Path(ANOTHER_NODE_FILE_NAME).name, + new_value=THIS_NODE_FILE_NAME, + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + ), + exp_new_get_value=DOWNLOAD_FILE_DIR + / "file_port_link" + / Path(THIS_NODE_FILE_NAME).name, ), - exp_value_type=(float), - exp_value_converter=float, - exp_value=PortLink( - nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", - output="the_output_of_that_node", + id="file node link type gets set back on store", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "number", + key="number_port_link", + value={ + "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + "output": "the_output_of_that_node", + }, + ), + exp_value_type=(float), + exp_value_converter=float, + exp_value=PortLink( + nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + output="the_output_of_that_node", + ), + exp_get_value=562.45, + new_value=None, + exp_new_value=None, + exp_new_get_value=None, ), - exp_get_value=562.45, - new_value=None, - exp_new_value=None, - exp_new_get_value=None, + id="number node link type gets reset", ), ], ) @@ -278,10 +388,17 @@ async def test_valid_port( new_value: Union[int, float, bool, str, Path], exp_new_value: Union[int, float, bool, str, Path, FileLink], exp_new_get_value: Union[int, float, bool, str, Path], + this_node_file: Path, + another_node_file: Path, ): class FakeNodePorts: async def get(self, key): - return exp_get_value + # this gets called when a node links to another node we return the get value but for files it needs to be a real one + return ( + another_node_file + if port_cfg["type"].startswith("data:") + else exp_get_value + ) async def _node_ports_creator_cb(self, node_uuid: str): return FakeNodePorts() @@ -309,7 +426,7 @@ async def save_to_db_cb(self, node_ports): if exp_get_value is None: assert await port.get() == None else: - assert await port.get() == exp_value_converter(exp_get_value) + assert await port.get() == exp_get_value # set a new value await port.set(new_value) @@ -317,7 +434,7 @@ async def save_to_db_cb(self, node_ports): if exp_new_get_value is None: assert await port.get() == None else: - assert await port.get() == exp_value_converter(exp_new_get_value) + assert await port.get() == exp_new_get_value @pytest.mark.parametrize( From 49d44fc4acd7eb1b505ce737acdb7ff6a082454e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sat, 5 Dec 2020 23:50:23 +0100 Subject: [PATCH 33/74] fix file to key map --- .../simcore_sdk/node_ports_v2/port_utils.py | 10 +- packages/simcore-sdk/tests/unit/test_port.py | 92 ++++++++++++++++++- 2 files changed, 98 insertions(+), 4 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py index 36caea4fb583..6b1a02a0eaf0 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -86,9 +86,17 @@ async def pull_file_from_download_link( downloaded_file = await filemanager.download_file_from_link( value.download_link, local_path, - file_name=next(iter(fileToKeyMap)) if fileToKeyMap else None, ) + # if a file alias is present use it to rename the file accordingly + if fileToKeyMap: + renamed_file = local_path / next(iter(fileToKeyMap)) + if downloaded_file != renamed_file: + if renamed_file.exists(): + renamed_file.unlink() + shutil.move(downloaded_file, renamed_file) + downloaded_file = renamed_file + return downloaded_file diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index 37b4fa471787..3d9ed450554c 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -260,7 +260,29 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: exp_new_value=None, exp_new_get_value=None, ), - id="file link with payload that get reset", + id="file link with payload that gets reset", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="some_file_with_file_to_key_map", + fileToKeyMap={ + "a_new_fancy_name.csv": "some_file_with_file_to_key_map" + }, + value={"store": "0", "path": THIS_NODE_FILE_NAME}, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=FileLink(store="0", path=THIS_NODE_FILE_NAME), + exp_get_value=DOWNLOAD_FILE_DIR + / "some_file_with_file_to_key_map" + / "a_new_fancy_name.csv", + new_value=None, + exp_new_value=None, + exp_new_get_value=None, + ), + id="file link with fileToKeyMap with payload that gets reset", ), pytest.param( *PortParams( @@ -269,7 +291,7 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: key="some_file_on_datcore", value={ "store": "1", - "path": THIS_NODE_FILE_NAME, + "path": f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", "dataset": "some blahblah", "label": "some blahblah", }, @@ -278,7 +300,7 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: exp_value_converter=Path, exp_value=FileLink( store="1", - path=THIS_NODE_FILE_NAME, + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", dataset="some blahblah", label="some blahblah", ), @@ -324,6 +346,37 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: ), id="download link file type gets set back on store", ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="download_link_with_file_to_key", + fileToKeyMap={ + "a_cool_file_type.zip": "download_link_with_file_to_key" + }, + value={ + "downloadLink": "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=DownloadLink( + downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" + ), + exp_get_value=DOWNLOAD_FILE_DIR + / "download_link_with_file_to_key" + / "a_cool_file_type.zip", + new_value=THIS_NODE_FILE_NAME, + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + ), + exp_new_get_value=DOWNLOAD_FILE_DIR + / "download_link_with_file_to_key" + / "a_cool_file_type.zip", + ), + id="download link file type with filetokeymap gets set back on store", + ), pytest.param( *PortParams( port_cfg=create_valid_port_config( @@ -354,6 +407,39 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: ), id="file node link type gets set back on store", ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="file_port_link_with_file_to_key_map", + fileToKeyMap={ + "a_cool_file_type.zip": "file_port_link_with_file_to_key_map" + }, + value={ + "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + "output": "the_output_of_that_node", + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=PortLink( + nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + output="the_output_of_that_node", + ), + exp_get_value=DOWNLOAD_FILE_DIR + / "file_port_link_with_file_to_key_map" + / "a_cool_file_type.zip", + new_value=THIS_NODE_FILE_NAME, + exp_new_value=FileLink( + store="0", + path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + ), + exp_new_get_value=DOWNLOAD_FILE_DIR + / "file_port_link_with_file_to_key_map" + / "a_cool_file_type.zip", + ), + id="file node link type with file to key map gets set back on store", + ), pytest.param( *PortParams( port_cfg=create_valid_port_config( From 73580e7d0e5a47ad7ac43b1ebe8c4b7d5c7d4796 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 6 Dec 2020 00:08:04 +0100 Subject: [PATCH 34/74] ensure if a file is already there it does not create issues --- packages/simcore-sdk/tests/unit/test_port.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index 3d9ed450554c..f7c7f93a14fd 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -509,6 +509,12 @@ async def save_to_db_cb(self, node_ports): assert port._py_value_converter == exp_value_converter assert port.value == exp_value + + if isinstance(exp_get_value, Path): + # if it's a file let's create one there already + exp_get_value.parent.mkdir(parents=True, exist_ok=True) + exp_get_value.touch() + if exp_get_value is None: assert await port.get() == None else: @@ -517,6 +523,11 @@ async def save_to_db_cb(self, node_ports): # set a new value await port.set(new_value) assert port.value == exp_new_value + + if isinstance(exp_new_get_value, Path): + # if it's a file let's create one there already + exp_new_get_value.parent.mkdir(parents=True, exist_ok=True) + exp_new_get_value.touch() if exp_new_get_value is None: assert await port.get() == None else: From 5540642051ba1d01ebf04ea0be9dd6411caa7316 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Sun, 6 Dec 2020 00:19:45 +0100 Subject: [PATCH 35/74] adding test of port mapping --- .../simcore_sdk/node_ports_v2/ports_mapping.py | 3 +++ .../simcore-sdk/tests/unit/test_port_mapping.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 packages/simcore-sdk/tests/unit/test_port_mapping.py diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py index 1349fed49d06..48c807e968c1 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py @@ -29,6 +29,9 @@ def items(self): def values(self): return self.__root__.values() + def __len__(self): + return self.__root__.__len__() + class InputsList(PortsMapping): __root__: Dict[PortKey, Port] diff --git a/packages/simcore-sdk/tests/unit/test_port_mapping.py b/packages/simcore-sdk/tests/unit/test_port_mapping.py new file mode 100644 index 000000000000..c6ee78a9b613 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_port_mapping.py @@ -0,0 +1,15 @@ +from typing import Any, Dict, Type, Union + +import pytest +from _pytest.mark import param +from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList + + +@pytest.mark.parametrize("port_class", [InputsList, OutputsList]) +def test_empty_ports_mapping_supports_dict_access( + port_class: Type[Union[InputsList, OutputsList]] +): + instance = port_class(**{"__root__": {}}) + assert not instance.items() + assert not instance.values() + assert len(instance) == 0 From 08e437837c9e8576fc7d3367e14b9bb11643b7a3 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 08:50:36 +0100 Subject: [PATCH 36/74] add some typical coveragerc definitions --- .coveragerc | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 786ba6c85c7e..21400449ce48 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,27 @@ [run] branch = True -omit = +omit = */tests/* */generated_code/* parallel = True + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +ignore_errors = True +show_missing = True From 128066a4042e7b6310867add9e63b37eb01113a4 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 09:45:17 +0100 Subject: [PATCH 37/74] use trick to use fixtures inside parametrization --- packages/simcore-sdk/tests/unit/test_port.py | 258 +++++++++++-------- 1 file changed, 156 insertions(+), 102 deletions(-) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index f7c7f93a14fd..e59dc75e2419 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -3,13 +3,13 @@ # pylint:disable=redefined-outer-name # pylint:disable=no-member # pylint:disable=protected-access +# pylint:disable=too-many-arguments import re import shutil import tempfile from asyncio import Future from collections import namedtuple from pathlib import Path -from random import randint from typing import Any, Dict, Optional, Type, Union import pytest @@ -22,29 +22,78 @@ from yarl import URL -@pytest.fixture(scope="module") +##################### HELPERS +def camel_to_snake(name): + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() + + +PortParams = namedtuple( + "PortParams", + "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", +) + + +def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: + valid_config = { + "key": f"some_{conf_type}", + "label": "some label", + "description": "some description", + "type": conf_type, + "displayOrder": 2.3, + } + valid_config.update(kwargs) + return valid_config + + +def this_node_file_name() -> Path: + return Path(tempfile.gettempdir(), "this_node_file.txt") + + +def another_node_file_name() -> Path: + return Path(tempfile.gettempdir(), "another_node_file.txt") + + +def download_file_folder_name() -> Path: + return Path(tempfile.gettempdir(), "simcorefiles") + + def project_id() -> str: return "cd0d8dbb-3263-44dc-921c-49c075ac0dd9" -@pytest.fixture(scope="module") def node_uuid() -> str: return "609b7af4-6861-4aa7-a16e-730ea8125190" -@pytest.fixture(scope="module") def user_id() -> str: - return str(randint(1, 666)) + return "666" -THIS_NODE_FILE_NAME: str = f"{tempfile.gettempdir()}/this_node_file.txt" -DOWNLOAD_FILE_DIR: Path = Path(tempfile.gettempdir(), "simcorefiles") -ANOTHER_NODE_FILE_NAME: Path = Path(tempfile.gettempdir(), "another_node_file.txt") +def simcore_store_id() -> str: + return "0" + + +def datcore_store_id() -> str: + return "1" + + +##################### FIXTURES @pytest.fixture -def this_node_file() -> Path: - file_path = Path(THIS_NODE_FILE_NAME) +async def mock_upload_file(mocker): + mock = mocker.patch( + "simcore_sdk.node_ports.filemanager.upload_file", + return_value=Future(), + ) + mock.return_value.set_result(simcore_store_id()) + yield mock + + +@pytest.fixture +def this_node_file(tmp_path: Path) -> Path: + file_path = this_node_file_name() file_path.write_text("some dummy data") assert file_path.exists() yield file_path @@ -54,7 +103,7 @@ def this_node_file() -> Path: @pytest.fixture def another_node_file() -> Path: - file_path = Path(tempfile.gettempdir(), "another_node_file.txt") + file_path = another_node_file_name() file_path.write_text("some dummy data") assert file_path.exists() yield file_path @@ -63,20 +112,42 @@ def another_node_file() -> Path: @pytest.fixture -def downloaded_file_folder() -> Path: - destination_path = DOWNLOAD_FILE_DIR +def download_file_folder() -> Path: + destination_path = download_file_folder_name() + destination_path.mkdir(parents=True, exist_ok=True) yield destination_path if destination_path.exists(): shutil.rmtree(destination_path) +@pytest.fixture(scope="module", name="project_id") +def project_id_fixture() -> str: + """NOTE: since pytest does not allow to use fixtures inside parametrizations, + this trick allows to re-use the same function in a fixture with a same "fixture" name""" + return project_id() + + +@pytest.fixture(scope="module", name="node_uuid") +def node_uuid_fixture() -> str: + """NOTE: since pytest does not allow to use fixtures inside parametrizations, + this trick allows to re-use the same function in a fixture with a same "fixture" name""" + return node_uuid() + + +@pytest.fixture(scope="module", name="user_id") +def user_id_fixture() -> str: + """NOTE: since pytest does not allow to use fixtures inside parametrizations, + this trick allows to re-use the same function in a fixture with a same "fixture" name""" + return user_id() + + @pytest.fixture async def mock_download_file( monkeypatch, this_node_file: Path, project_id: str, node_uuid: str, - downloaded_file_folder: Path, + download_file_folder: Path, ): async def mock_download_file_from_link( download_link: URL, @@ -84,7 +155,7 @@ async def mock_download_file_from_link( session: Optional[ClientSession] = None, file_name: Optional[str] = None, ) -> Path: - assert str(local_folder).startswith(str(DOWNLOAD_FILE_DIR)) + assert str(local_folder).startswith(str(download_file_folder)) destination_path = local_folder / this_node_file.name destination_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy(this_node_file, destination_path) @@ -97,16 +168,6 @@ async def mock_download_file_from_link( ) -@pytest.fixture -async def mock_upload_file(mocker): - mock = mocker.patch( - "simcore_sdk.node_ports.filemanager.upload_file", - return_value=Future(), - ) - mock.return_value.set_result("0") - yield mock - - @pytest.fixture(autouse=True) def node_ports_config( loop, @@ -116,36 +177,18 @@ def node_ports_config( project_id: str, user_id: str, node_uuid: str, + this_node_file: Path, + another_node_file: Path, + download_file_folder: Path, ): + """this module main fixture""" + config.USER_ID = user_id config.PROJECT_ID = project_id config.NODE_UUID = node_uuid config.STORAGE_ENDPOINT = "storage:8080" -def camel_to_snake(name): - name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) - return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() - - -PortParams = namedtuple( - "PortParams", - "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", -) - - -def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: - valid_config = { - "key": f"some_{conf_type}", - "label": "some label", - "description": "some description", - "type": conf_type, - "displayOrder": 2.3, - } - valid_config.update(kwargs) - return valid_config - - @pytest.mark.parametrize( "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", [ @@ -210,14 +253,14 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: exp_value_converter=Path, exp_value=None, exp_get_value=None, - new_value=THIS_NODE_FILE_NAME, + new_value=str(this_node_file_name()), exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", ), - exp_new_get_value=DOWNLOAD_FILE_DIR + exp_new_get_value=download_file_folder_name() / "no_file" - / Path(THIS_NODE_FILE_NAME).name, + / this_node_file_name().name, ), id="file type with no payload", ), @@ -226,20 +269,20 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: port_cfg=create_valid_port_config( "data:*/*", key="no_file_with_default", - defaultValue=THIS_NODE_FILE_NAME, + defaultValue=str(this_node_file_name()), ), exp_value_type=(Path, str), exp_value_converter=Path, exp_value=None, exp_get_value=None, - new_value=THIS_NODE_FILE_NAME, + new_value=this_node_file_name(), exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", ), - exp_new_get_value=DOWNLOAD_FILE_DIR + exp_new_get_value=download_file_folder_name() / "no_file_with_default" - / Path(THIS_NODE_FILE_NAME).name, + / this_node_file_name().name, ), id="file link with no payload and default value", ), @@ -248,14 +291,20 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: port_cfg=create_valid_port_config( "data:*/*", key="some_file", - value={"store": "0", "path": THIS_NODE_FILE_NAME}, + value={ + "store": simcore_store_id(), + "path": f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + }, ), exp_value_type=(Path, str), exp_value_converter=Path, - exp_value=FileLink(store="0", path=THIS_NODE_FILE_NAME), - exp_get_value=DOWNLOAD_FILE_DIR + exp_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + ), + exp_get_value=download_file_folder_name() / "some_file" - / Path(THIS_NODE_FILE_NAME).name, + / this_node_file_name().name, new_value=None, exp_new_value=None, exp_new_get_value=None, @@ -270,12 +319,18 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: fileToKeyMap={ "a_new_fancy_name.csv": "some_file_with_file_to_key_map" }, - value={"store": "0", "path": THIS_NODE_FILE_NAME}, + value={ + "store": simcore_store_id(), + "path": f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + }, ), exp_value_type=(Path, str), exp_value_converter=Path, - exp_value=FileLink(store="0", path=THIS_NODE_FILE_NAME), - exp_get_value=DOWNLOAD_FILE_DIR + exp_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + ), + exp_get_value=download_file_folder_name() / "some_file_with_file_to_key_map" / "a_new_fancy_name.csv", new_value=None, @@ -290,8 +345,8 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: "data:*/*", key="some_file_on_datcore", value={ - "store": "1", - "path": f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + "store": datcore_store_id(), + "path": f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", "dataset": "some blahblah", "label": "some blahblah", }, @@ -299,22 +354,22 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: exp_value_type=(Path, str), exp_value_converter=Path, exp_value=FileLink( - store="1", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + store=datcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", dataset="some blahblah", label="some blahblah", ), - exp_get_value=DOWNLOAD_FILE_DIR + exp_get_value=download_file_folder_name() / "some_file_on_datcore" - / Path(THIS_NODE_FILE_NAME).name, - new_value=THIS_NODE_FILE_NAME, + / this_node_file_name().name, + new_value=this_node_file_name(), exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", ), - exp_new_get_value=DOWNLOAD_FILE_DIR + exp_new_get_value=download_file_folder_name() / "some_file_on_datcore" - / Path(THIS_NODE_FILE_NAME).name, + / this_node_file_name().name, ), id="file link with payload on store 1", ), @@ -332,17 +387,17 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: exp_value=DownloadLink( downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" ), - exp_get_value=DOWNLOAD_FILE_DIR + exp_get_value=download_file_folder_name() / "download_link" - / Path(THIS_NODE_FILE_NAME).name, - new_value=THIS_NODE_FILE_NAME, + / this_node_file_name().name, + new_value=this_node_file_name(), exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", ), - exp_new_get_value=DOWNLOAD_FILE_DIR + exp_new_get_value=download_file_folder_name() / "download_link" - / Path(THIS_NODE_FILE_NAME).name, + / this_node_file_name().name, ), id="download link file type gets set back on store", ), @@ -363,15 +418,15 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: exp_value=DownloadLink( downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" ), - exp_get_value=DOWNLOAD_FILE_DIR + exp_get_value=download_file_folder_name() / "download_link_with_file_to_key" / "a_cool_file_type.zip", - new_value=THIS_NODE_FILE_NAME, + new_value=this_node_file_name(), exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", ), - exp_new_get_value=DOWNLOAD_FILE_DIR + exp_new_get_value=download_file_folder_name() / "download_link_with_file_to_key" / "a_cool_file_type.zip", ), @@ -393,17 +448,17 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", output="the_output_of_that_node", ), - exp_get_value=DOWNLOAD_FILE_DIR + exp_get_value=download_file_folder_name() / "file_port_link" - / Path(ANOTHER_NODE_FILE_NAME).name, - new_value=THIS_NODE_FILE_NAME, + / another_node_file_name().name, + new_value=this_node_file_name(), exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", ), - exp_new_get_value=DOWNLOAD_FILE_DIR + exp_new_get_value=download_file_folder_name() / "file_port_link" - / Path(THIS_NODE_FILE_NAME).name, + / this_node_file_name().name, ), id="file node link type gets set back on store", ), @@ -426,15 +481,15 @@ def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", output="the_output_of_that_node", ), - exp_get_value=DOWNLOAD_FILE_DIR + exp_get_value=download_file_folder_name() / "file_port_link_with_file_to_key_map" / "a_cool_file_type.zip", - new_value=THIS_NODE_FILE_NAME, + new_value=this_node_file_name(), exp_new_value=FileLink( - store="0", - path=f"cd0d8dbb-3263-44dc-921c-49c075ac0dd9/609b7af4-6861-4aa7-a16e-730ea8125190/{Path(THIS_NODE_FILE_NAME).name}", + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", ), - exp_new_get_value=DOWNLOAD_FILE_DIR + exp_new_get_value=download_file_folder_name() / "file_port_link_with_file_to_key_map" / "a_cool_file_type.zip", ), @@ -474,7 +529,6 @@ async def test_valid_port( new_value: Union[int, float, bool, str, Path], exp_new_value: Union[int, float, bool, str, Path, FileLink], exp_new_get_value: Union[int, float, bool, str, Path], - this_node_file: Path, another_node_file: Path, ): class FakeNodePorts: From f2bef9b018dd0749c72e881a6a32452979f51030 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 11:30:16 +0100 Subject: [PATCH 38/74] 100% test on port_mapping --- .../node_ports_v2/ports_mapping.py | 3 + packages/simcore-sdk/tests/unit/test_port.py | 1 + .../tests/unit/test_port_mapping.py | 60 ++++++++++++++++--- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py index 48c807e968c1..9ffa64760686 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py @@ -23,6 +23,9 @@ def __getitem__(self, key: Union[int, PortKey]) -> Port: def __iter__(self): return iter(self.__root__) + def keys(self): + return self.__root__.keys() + def items(self): return self.__root__.items() diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index e59dc75e2419..074f85f474b0 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -189,6 +189,7 @@ def node_ports_config( config.STORAGE_ENDPOINT = "storage:8080" +##################### TESTS @pytest.mark.parametrize( "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", [ diff --git a/packages/simcore-sdk/tests/unit/test_port_mapping.py b/packages/simcore-sdk/tests/unit/test_port_mapping.py index c6ee78a9b613..d1e64dbc0e2a 100644 --- a/packages/simcore-sdk/tests/unit/test_port_mapping.py +++ b/packages/simcore-sdk/tests/unit/test_port_mapping.py @@ -1,15 +1,59 @@ from typing import Any, Dict, Type, Union import pytest -from _pytest.mark import param +from simcore_sdk.node_ports.exceptions import UnboundPortError +from simcore_sdk.node_ports_v2.port import Port from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList +##################### HELPERS +def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: + valid_config = { + "key": f"some_{conf_type}", + "label": "some label", + "description": "some description", + "type": conf_type, + "displayOrder": 2.3, + } + valid_config.update(kwargs) + return valid_config + + +##################### TESTS +@pytest.mark.parametrize("port_class", [InputsList, OutputsList]) +def test_empty_ports_mapping(port_class: Type[Union[InputsList, OutputsList]]): + port_mapping = port_class(**{"__root__": {}}) + assert not port_mapping.items() + assert not port_mapping.values() + assert not port_mapping.keys() + assert len(port_mapping) == 0 + for port_key in port_mapping: + # it should be empty + assert True, f"should be empty, got {port_key}" + + @pytest.mark.parametrize("port_class", [InputsList, OutputsList]) -def test_empty_ports_mapping_supports_dict_access( - port_class: Type[Union[InputsList, OutputsList]] -): - instance = port_class(**{"__root__": {}}) - assert not instance.items() - assert not instance.values() - assert len(instance) == 0 +def test_filled_ports_mapping(port_class: Type[Union[InputsList, OutputsList]]): + port_cfgs: Dict[str, Any] = {} + for t in ["integer", "number", "boolean", "string"]: + port = create_valid_port_config(t) + port_cfgs[port["key"]] = port + port_cfgs["some_file"] = create_valid_port_config("data:*/*", key="some_file") + port_mapping = port_class(**{"__root__": port_cfgs}) + + assert len(port_mapping) == len(port_cfgs) + for port_key, port_value in port_mapping.items(): + assert port_key in port_mapping + assert port_key in port_cfgs + + # just to make use of the variable and check the pydantic overloads are working correctly + assert port_mapping[port_key] == port_value + + for index, port_key in enumerate(port_cfgs): + assert port_mapping[index] == port_mapping[port_key] + + with pytest.raises(UnboundPortError): + _ = port_mapping[len(port_cfgs)] + + with pytest.raises(UnboundPortError): + _ = port_mapping["whatever"] From f62e78e13d3cecb6254c9eca70a88063223eec3e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 11:48:05 +0100 Subject: [PATCH 39/74] move fixtures to base conftest move mocks to base test folder --- packages/simcore-sdk/tests/conftest.py | 16 ++++++++++++++++ .../simcore-sdk/tests/integration/conftest.py | 10 ---------- .../{integration => }/mock/default_config.json | 0 .../{integration => }/mock/empty_config.json | 4 ++-- .../tests/unit/test_serialization_v2.py | 6 ++++++ 5 files changed, 24 insertions(+), 12 deletions(-) rename packages/simcore-sdk/tests/{integration => }/mock/default_config.json (100%) rename packages/simcore-sdk/tests/{integration => }/mock/empty_config.json (84%) create mode 100644 packages/simcore-sdk/tests/unit/test_serialization_v2.py diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index 3877097b3ab1..21e5ffff91b8 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -2,8 +2,10 @@ # pylint:disable=redefined-outer-name import sys from pathlib import Path +from typing import Dict import pytest +import sqlalchemy as sa ## HELPERS current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -43,3 +45,17 @@ def env_devel_file(osparc_simcore_root_dir) -> Path: env_devel_fpath = osparc_simcore_root_dir / ".env-devel" assert env_devel_fpath.exists() return env_devel_fpath + + +@pytest.fixture(scope="session") +def default_configuration_file() -> Path: + path = current_dir / "mock" / "default_config.json" + assert path.exists() + return path + + +@pytest.fixture(scope="session") +def empty_configuration_file() -> Path: + return current_dir / "mock" / "empty_config.json" + assert path.exists() + return path diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index 725843be8595..f9dcbb579bf7 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -79,16 +79,6 @@ def create(file_path: Path, project: str = None, node: str = None): yield create -@pytest.fixture -def default_configuration_file() -> Path: - return current_dir / "mock" / "default_config.json" - - -@pytest.fixture -def empty_configuration_file() -> Path: - return current_dir / "mock" / "empty_config.json" - - @pytest.fixture() def default_configuration( nodeports_config, diff --git a/packages/simcore-sdk/tests/integration/mock/default_config.json b/packages/simcore-sdk/tests/mock/default_config.json similarity index 100% rename from packages/simcore-sdk/tests/integration/mock/default_config.json rename to packages/simcore-sdk/tests/mock/default_config.json diff --git a/packages/simcore-sdk/tests/integration/mock/empty_config.json b/packages/simcore-sdk/tests/mock/empty_config.json similarity index 84% rename from packages/simcore-sdk/tests/integration/mock/empty_config.json rename to packages/simcore-sdk/tests/mock/empty_config.json index 943566ea71c2..b5f9c76d253b 100644 --- a/packages/simcore-sdk/tests/integration/mock/empty_config.json +++ b/packages/simcore-sdk/tests/mock/empty_config.json @@ -6,8 +6,8 @@ "outputs" : { } }, - "inputs": { + "inputs": { }, "outputs": { } -} \ No newline at end of file +} diff --git a/packages/simcore-sdk/tests/unit/test_serialization_v2.py b/packages/simcore-sdk/tests/unit/test_serialization_v2.py new file mode 100644 index 000000000000..3da135127539 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_serialization_v2.py @@ -0,0 +1,6 @@ +async def test_create_nodeports_from_db(): + pass + + +async def test_save_nodeports_to_db(): + pass From 2e5f4c6d9f83b89d81c72268e6265f5f4486057a Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 11:56:13 +0100 Subject: [PATCH 40/74] tests refactor --- packages/simcore-sdk/tests/conftest.py | 14 ++++++- .../simcore-sdk/tests/integration/conftest.py | 37 +++++++------------ .../tests/integration/test_dbmanager.py | 4 +- packages/simcore-sdk/tests/unit/test_port.py | 2 +- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index 21e5ffff91b8..334616ffc8d8 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -6,6 +6,7 @@ import pytest import sqlalchemy as sa +from simcore_sdk.node_ports import config as node_config ## HELPERS current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -56,6 +57,17 @@ def default_configuration_file() -> Path: @pytest.fixture(scope="session") def empty_configuration_file() -> Path: - return current_dir / "mock" / "empty_config.json" + path = current_dir / "mock" / "empty_config.json" assert path.exists() return path + + +@pytest.fixture(scope="module") +def node_ports_config( + postgres_dsn: Dict[str, str], minio_config: Dict[str, str] +) -> None: + node_config.POSTGRES_DB = postgres_dsn["database"] + node_config.POSTGRES_ENDPOINT = f"{postgres_dsn['host']}:{postgres_dsn['port']}" + node_config.POSTGRES_USER = postgres_dsn["user"] + node_config.POSTGRES_PW = postgres_dsn["password"] + node_config.BUCKET = minio_config["bucket_name"] diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index f9dcbb579bf7..618efc4b206a 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -20,23 +20,22 @@ current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent -@pytest.fixture -def nodeports_config( - postgres_dsn: Dict[str, str], minio_config: Dict[str, str] -) -> None: - node_config.POSTGRES_DB = postgres_dsn["database"] - node_config.POSTGRES_ENDPOINT = f"{postgres_dsn['host']}:{postgres_dsn['port']}" - node_config.POSTGRES_USER = postgres_dsn["user"] - node_config.POSTGRES_PW = postgres_dsn["password"] - node_config.BUCKET = minio_config["bucket_name"] - - @pytest.fixture def user_id() -> int: # see fixtures/postgres.py yield 1258 +@pytest.fixture +def project_id() -> str: + return str(uuid.uuid4()) + + +@pytest.fixture +def node_uuid() -> str: + return str(uuid.uuid4()) + + @pytest.fixture def s3_simcore_location() -> str: yield np_helpers.SIMCORE_STORE @@ -57,16 +56,6 @@ async def filemanager_cfg( yield -@pytest.fixture -def project_id() -> str: - return str(uuid.uuid4()) - - -@pytest.fixture -def node_uuid() -> str: - return str(uuid.uuid4()) - - @pytest.fixture def file_uuid(project_id: str, node_uuid: str) -> Callable: def create(file_path: Path, project: str = None, node: str = None): @@ -81,7 +70,7 @@ def create(file_path: Path, project: str = None, node: str = None): @pytest.fixture() def default_configuration( - nodeports_config, + node_ports_config, bucket, postgres_session: sa.orm.session.Session, default_configuration_file: Path, @@ -130,7 +119,7 @@ def create_store_link( @pytest.fixture(scope="function") def special_configuration( - nodeports_config: None, + node_ports_config: None, bucket: str, postgres_session: sa.orm.session.Session, empty_configuration_file: Path, @@ -163,7 +152,7 @@ def create_config( @pytest.fixture(scope="function") def special_2nodes_configuration( - nodeports_config: None, + node_ports_config: None, bucket: str, postgres_session: sa.orm.session.Session, empty_configuration_file: Path, diff --git a/packages/simcore-sdk/tests/integration/test_dbmanager.py b/packages/simcore-sdk/tests/integration/test_dbmanager.py index a5a5fe370cc2..b85d3debda5b 100644 --- a/packages/simcore-sdk/tests/integration/test_dbmanager.py +++ b/packages/simcore-sdk/tests/integration/test_dbmanager.py @@ -19,7 +19,7 @@ async def test_db_manager_read_config( loop: asyncio.events.AbstractEventLoop, - nodeports_config: None, + node_ports_config: None, default_configuration: Dict, ): db_manager = DBManager() @@ -33,7 +33,7 @@ async def test_db_manager_read_config( async def test_db_manager_write_config( loop: asyncio.events.AbstractEventLoop, - nodeports_config: None, + node_ports_config: None, special_configuration: Callable, default_configuration_file: Path, ): diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index 074f85f474b0..e14c4c349850 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -169,7 +169,7 @@ async def mock_download_file_from_link( @pytest.fixture(autouse=True) -def node_ports_config( +def common_fixtures( loop, storage_v0_subsystem_mock, mock_download_file, From 7b36c1491ef9ede16ea3d06668390e19be1cbb93 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 16:29:14 +0100 Subject: [PATCH 41/74] tests serialization2 --- .../src/simcore_sdk/node_ports/dbmanager.py | 1 - .../src/simcore_sdk/node_ports_v2/port.py | 13 ++- .../node_ports_v2/serialization_v2.py | 16 ++- packages/simcore-sdk/tests/conftest.py | 11 +- .../tests/mock/default_config.json | 107 +++++++++--------- .../simcore-sdk/tests/mock/empty_config.json | 17 +-- .../tests/unit/test_serialization_v2.py | 95 +++++++++++++++- 7 files changed, 179 insertions(+), 81 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py index f038e2d5832e..deec34502452 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py @@ -124,7 +124,6 @@ async def get_ports_configuration_from_node_uuid(self, node_uuid: str) -> str: node = await _get_node_from_db(node_uuid, connection) node_json_config = json.dumps( { - "version": "0.1", "schema": node.schema, "inputs": node.inputs, "outputs": node.outputs, diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index d8f0c7afb346..41881216eb75 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -30,6 +30,7 @@ class Port(ServiceProperty): _py_value_type: Tuple[Type[ItemConcreteValue]] = PrivateAttr() _py_value_converter: Type[ItemConcreteValue] = PrivateAttr() _node_ports = PrivateAttr() + _used_default_value: bool = PrivateAttr(False) @validator("value", always=True) @classmethod @@ -41,13 +42,11 @@ def ensure_value(cls, v, values): raise ValueError( f"[{values['property_type']}] must follow {FileLink.schema()}, {DownloadLink.schema()} or {PortLink.schema()}" ) - elif v is None and values.get("default_value"): - return values["default_value"] return v def __init__(self, **data): super().__init__(**data) - # let's define the converter + self._py_value_type = ( (Path, str) if port_utils.is_file_type(self.property_type) @@ -58,6 +57,13 @@ def __init__(self, **data): if port_utils.is_file_type(self.property_type) else TYPE_TO_PYTYPE[self.property_type] ) + if ( + self.value is None + and self.default_value is not None + and not port_utils.is_file_type(self.property_type) + ): + self.value = self.default_value + self._used_default_value = True async def get(self) -> ItemConcreteValue: log.debug( @@ -113,4 +119,5 @@ async def set(self, new_value: ItemConcreteValue): ) self.value = converted_value + self._used_default_value = False await self._node_ports.save_to_db_cb(self._node_ports) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py index 1ff7d116ed3c..8748a4fd31d0 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -7,7 +7,7 @@ from ..node_ports.exceptions import InvalidProtocolError from .nodeports_v2 import Nodeports -NODE_REQUIORED_KEYS: List[str] = { +NODE_REQUIRED_KEYS: List[str] = { "schema", "inputs", "outputs", @@ -20,7 +20,7 @@ async def create_nodeports_from_db( """creates a nodeport object from a row from comp_tasks""" row: RowProxy = await db_manager.get_ports_configuration_from_node_uuid(node_uuid) port_cfg = json.loads(row) - if any(k not in port_cfg for k in NODE_REQUIORED_KEYS): + if any(k not in port_cfg for k in NODE_REQUIRED_KEYS): raise InvalidProtocolError( port_cfg, "nodeport in comp_task does not follow protocol" ) @@ -45,12 +45,12 @@ async def create_nodeports_from_db( node_uuid=node_uuid, save_to_db_cb=save_nodeports_to_db, node_port_creator_cb=create_nodeports_from_db, - auto_update=auto_update + auto_update=auto_update, ) return ports -async def save_nodeports_to_db(nodeports: Nodeports): +async def save_nodeports_to_db(nodeports: Nodeports) -> None: _nodeports_cfg = nodeports.dict( include={"internal_inputs", "internal_outputs"}, by_alias=True, @@ -68,8 +68,12 @@ async def save_nodeports_to_db(nodeports: Nodeports): if k not in ["key", "value"] } port_cfg["schema"][port_type][port_key] = key_schema - # payload - port_cfg[port_type][port_key] = port_values["value"] + # payload (only if default value was not used) + # pylint: disable=protected-access + if not getattr(nodeports, f"internal_{port_type}")[ + port_key + ]._used_default_value: + port_cfg[port_type][port_key] = port_values["value"] await nodeports.db_manager.write_ports_configuration( json.dumps(port_cfg), nodeports.node_uuid diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index 334616ffc8d8..ec0f1bcdf91d 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -1,11 +1,12 @@ +import json + # pylint:disable=unused-argument # pylint:disable=redefined-outer-name import sys from pathlib import Path -from typing import Dict +from typing import Any, Dict import pytest -import sqlalchemy as sa from simcore_sdk.node_ports import config as node_config ## HELPERS @@ -55,6 +56,12 @@ def default_configuration_file() -> Path: return path +@pytest.fixture(scope="session") +def default_configuration(default_configuration_file: Path) -> Dict[str, Any]: + config = json.loads(default_configuration_file.read_text()) + return config + + @pytest.fixture(scope="session") def empty_configuration_file() -> Path: path = current_dir / "mock" / "empty_config.json" diff --git a/packages/simcore-sdk/tests/mock/default_config.json b/packages/simcore-sdk/tests/mock/default_config.json index 392c27aab8c8..35bdfd9cfc4e 100644 --- a/packages/simcore-sdk/tests/mock/default_config.json +++ b/packages/simcore-sdk/tests/mock/default_config.json @@ -1,60 +1,59 @@ { - "version": "0.1", - "schema": { - "inputs": { - "in_1": { - "displayOrder": 0, - "label": "computational data", - "description": "these are computed data out of a pipeline", - "type": "data:*/*", - "defaultValue": null, - "fileToKeyMap": { - "input1.txt": "in_1" - }, - "widget": null - }, - "in_5": { - "displayOrder": 2, - "label": "some number", - "description": "numbering things", - "type": "integer", - "defaultValue": 666, - "fileToKeyMap": {}, - "widget": null - } - }, - "outputs": { - "out_1": { - "displayOrder": 0, - "label": "some boolean output", - "description": "could be true or false...", - "type": "boolean", - "defaultValue": null, - "fileToKeyMap": {}, - "widget": null - }, - "out_2": { - "displayOrder": 1, - "label": "some file output", - "description": "could be anything...", - "type": "data:*/*", - "defaultValue": null, - "fileToKeyMap": {}, - "widget": null - } - } - }, + "schema": { "inputs": { - "in_1": { - "nodeUuid": "793bc431-b935-4f9f-8b04-e4117640e9b6", - "output": "outFile" - } + "in_1": { + "displayOrder": 0, + "label": "computational data", + "description": "these are computed data out of a pipeline", + "type": "data:*/*", + "defaultValue": null, + "fileToKeyMap": { + "input1.txt": "in_1" + }, + "widget": null + }, + "in_5": { + "displayOrder": 2, + "label": "some number", + "description": "numbering things", + "type": "integer", + "defaultValue": 666, + "fileToKeyMap": {}, + "widget": null + } }, "outputs": { - "out_1": false, - "out_2": { - "store": "z43-s3", - "path": "/simcore/outputControllerOut.dat" - } + "out_1": { + "displayOrder": 0, + "label": "some boolean output", + "description": "could be true or false...", + "type": "boolean", + "defaultValue": null, + "fileToKeyMap": {}, + "widget": null + }, + "out_2": { + "displayOrder": 1, + "label": "some file output", + "description": "could be anything...", + "type": "data:*/*", + "defaultValue": null, + "fileToKeyMap": {}, + "widget": null + } + } + }, + "inputs": { + "in_1": { + "nodeUuid": "793bc431-b935-4f9f-8b04-e4117640e9b6", + "output": "outFile" + } + }, + "outputs": { + "out_1": false, + "out_2": { + "store": "z43-s3", + "path": "/simcore/outputControllerOut.dat" } + } } diff --git a/packages/simcore-sdk/tests/mock/empty_config.json b/packages/simcore-sdk/tests/mock/empty_config.json index b5f9c76d253b..33f26d13e6a1 100644 --- a/packages/simcore-sdk/tests/mock/empty_config.json +++ b/packages/simcore-sdk/tests/mock/empty_config.json @@ -1,13 +1,8 @@ { - "version":"0.1", - "schema": { - "inputs": { - }, - "outputs" : { - } - }, - "inputs": { - }, - "outputs": { - } + "schema": { + "inputs": {}, + "outputs": {} + }, + "inputs": {}, + "outputs": {} } diff --git a/packages/simcore-sdk/tests/unit/test_serialization_v2.py b/packages/simcore-sdk/tests/unit/test_serialization_v2.py index 3da135127539..a3bf84fcdb21 100644 --- a/packages/simcore-sdk/tests/unit/test_serialization_v2.py +++ b/packages/simcore-sdk/tests/unit/test_serialization_v2.py @@ -1,6 +1,93 @@ -async def test_create_nodeports_from_db(): - pass +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +import asyncio +import json +from typing import Any, Dict +from uuid import uuid4 -async def test_save_nodeports_to_db(): - pass +import pytest +from simcore_sdk.node_ports.dbmanager import DBManager +from simcore_sdk.node_ports.exceptions import InvalidProtocolError +from simcore_sdk.node_ports_v2.serialization_v2 import ( + create_nodeports_from_db, + save_nodeports_to_db, +) +from sqlalchemy.exc import InvalidatePoolError + + +@pytest.fixture(scope="module") +def node_uuid() -> str: + return str(uuid4()) + + +@pytest.fixture(scope="function") +async def mock_db_manager( + loop: asyncio.AbstractEventLoop, + monkeypatch, + node_uuid: str, +): + def _mock_db_manager(port_cfg: Dict[str, Any]): + async def mock_get_ports_configuration_from_node_uuid(*args, **kwargs) -> str: + return json.dumps(port_cfg) + + async def mock_write_ports_configuration( + self, json_configuration: str, uuid: str + ): + assert json.loads(json_configuration) == port_cfg + assert uuid == node_uuid + + monkeypatch.setattr( + DBManager, + "get_ports_configuration_from_node_uuid", + mock_get_ports_configuration_from_node_uuid, + ) + monkeypatch.setattr( + DBManager, + "write_ports_configuration", + mock_write_ports_configuration, + ) + + yield _mock_db_manager + + +@pytest.mark.parametrize("auto_update", [True, False]) +async def test_create_nodeports_from_db( + mock_db_manager, + auto_update: bool, + node_uuid: str, + default_configuration: Dict[str, Any], +): + mock_db_manager(default_configuration) + db_manager = DBManager() + node_ports = await create_nodeports_from_db(db_manager, node_uuid, auto_update) + assert node_ports.db_manager == db_manager + assert node_ports.node_uuid == node_uuid + # pylint: disable=comparison-with-callable + assert node_ports.save_to_db_cb == save_nodeports_to_db + assert node_ports.node_port_creator_cb == create_nodeports_from_db + assert node_ports.auto_update == auto_update + + +async def test_create_nodeports_from_db_with_invalid_cfg( + mock_db_manager, + node_uuid: str, +): + invalid_config = {"bad_key": "bad_value"} + mock_db_manager(invalid_config) + db_manager = DBManager() + with pytest.raises(InvalidProtocolError): + node_ports = await create_nodeports_from_db(db_manager, node_uuid) + + +async def test_save_nodeports_to_db( + mock_db_manager, + node_uuid: str, + default_configuration: Dict[str, Any], +): + mock_db_manager(default_configuration) + db_manager = DBManager() + node_ports = await create_nodeports_from_db(db_manager, node_uuid) + + await save_nodeports_to_db(node_ports) From 43f7cbc4b5d1b68260bbeb610e7a4ee6c266555b Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 17:58:11 +0100 Subject: [PATCH 42/74] not far from 100% --- .../simcore_sdk/node_ports_v2/nodeports_v2.py | 2 + .../tests/helpers/utils_port_v2.py | 46 +++++ packages/simcore-sdk/tests/unit/conftest.py | 49 +++++ .../tests/unit/test_nodeports_v2.py | 178 ++++++++++++++++++ packages/simcore-sdk/tests/unit/test_port.py | 13 +- .../tests/unit/test_port_mapping.py | 15 +- .../tests/unit/test_serialization_v2.py | 50 +---- 7 files changed, 281 insertions(+), 72 deletions(-) create mode 100644 packages/simcore-sdk/tests/helpers/utils_port_v2.py create mode 100644 packages/simcore-sdk/tests/unit/conftest.py create mode 100644 packages/simcore-sdk/tests/unit/test_nodeports_v2.py diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 11eb971c6d77..6ef2c5af481f 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -59,6 +59,7 @@ async def get(self, item_key: str) -> ItemConcreteValue: async def set(self, item_key: str, item_value): try: await (await self.inputs)[item_key].set(item_value) + return except UnboundPortError: # not available try outputs pass @@ -83,6 +84,7 @@ async def _auto_update_from_db(self) -> None: self.internal_inputs = updated_node_ports.internal_inputs self.internal_outputs = updated_node_ports.internal_outputs # let's pass ourselves down + # pylint: disable=protected-access for input_key in self.internal_inputs: self.internal_inputs[input_key]._node_ports = self for output_key in self.internal_outputs: diff --git a/packages/simcore-sdk/tests/helpers/utils_port_v2.py b/packages/simcore-sdk/tests/helpers/utils_port_v2.py new file mode 100644 index 000000000000..19a8957668e5 --- /dev/null +++ b/packages/simcore-sdk/tests/helpers/utils_port_v2.py @@ -0,0 +1,46 @@ +from typing import Any, Dict, Optional, Type, Union + +from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList + + +def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: + valid_config = { + "key": f"some_{conf_type}", + "label": "some label", + "description": "some description", + "type": conf_type, + "displayOrder": 2.3, + } + valid_config.update(kwargs) + return valid_config + + +def create_valid_port_mapping( + mapping_class: Type[Union[InputsList, OutputsList]], + suffix: str, + file_to_key: Optional[str] = None, +) -> Type[Union[InputsList, OutputsList]]: + port_cfgs: Dict[str, Any] = {} + for t, v in { + "integer": 43, + "number": 45.6, + "boolean": True, + "string": "dfgjkhdf", + }.items(): + port = create_valid_port_config( + t, + key=f"{'input' if mapping_class==InputsList else 'output'}_{t}_{suffix}", + value=v, + ) + port_cfgs[port["key"]] = port + + key_for_file_port = ( + f"{'input' if mapping_class==InputsList else 'output'}_file_{suffix}" + ) + port_cfgs[key_for_file_port] = create_valid_port_config( + "data:*/*", + key=key_for_file_port, + fileToKeyMap={file_to_key: key_for_file_port} if file_to_key else None, + ) + port_mapping = mapping_class(**{"__root__": port_cfgs}) + return port_mapping diff --git a/packages/simcore-sdk/tests/unit/conftest.py b/packages/simcore-sdk/tests/unit/conftest.py new file mode 100644 index 000000000000..b1c60061e57b --- /dev/null +++ b/packages/simcore-sdk/tests/unit/conftest.py @@ -0,0 +1,49 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import asyncio +import json +from typing import Any, Callable, Dict +from uuid import uuid4 + +import pytest +from simcore_sdk.node_ports.dbmanager import DBManager + + +@pytest.fixture(scope="module") +def node_uuid() -> str: + return str(uuid4()) + + +@pytest.fixture(scope="function") +async def mock_db_manager( + loop: asyncio.AbstractEventLoop, + monkeypatch, + node_uuid: str, +) -> Callable: + def _mock_db_manager(port_cfg: Dict[str, Any]) -> DBManager: + async def mock_get_ports_configuration_from_node_uuid(*args, **kwargs) -> str: + return json.dumps(port_cfg) + + async def mock_write_ports_configuration( + self, json_configuration: str, uuid: str + ): + assert json.loads(json_configuration) == port_cfg + assert uuid == node_uuid + + monkeypatch.setattr( + DBManager, + "get_ports_configuration_from_node_uuid", + mock_get_ports_configuration_from_node_uuid, + ) + monkeypatch.setattr( + DBManager, + "write_ports_configuration", + mock_write_ports_configuration, + ) + + db_manager = DBManager() + return db_manager + + yield _mock_db_manager diff --git a/packages/simcore-sdk/tests/unit/test_nodeports_v2.py b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py new file mode 100644 index 000000000000..8e07756aec23 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py @@ -0,0 +1,178 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +from asyncio import Future +from pathlib import Path +from typing import Any, Callable, Dict + +import pytest +from simcore_sdk.node_ports.exceptions import PortNotFound, UnboundPortError +from simcore_sdk.node_ports_v2 import Nodeports, ports +from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList +from utils_port_v2 import create_valid_port_mapping + + +@pytest.mark.parametrize( + "auto_update", + [ + pytest.param(True, id="Autoupdate enabled"), + pytest.param(False, id="Autoupdate disabled"), + ], +) +async def test_nodeports_aut_updates( + mock_db_manager: Callable, + default_configuration: Dict[str, Any], + node_uuid: str, + auto_update: bool, +): + db_manager = mock_db_manager(default_configuration) + + original_inputs = create_valid_port_mapping(InputsList, suffix="original") + original_outputs = create_valid_port_mapping(OutputsList, suffix="original") + + updated_inputs = create_valid_port_mapping(InputsList, suffix="updated") + updated_outputs = create_valid_port_mapping(OutputsList, suffix="updated") + + async def mock_save_db_cb(*args, **kwargs): + pass + + async def mock_node_port_creator_cb(*args, **kwargs): + updated_node_ports = Nodeports( + inputs=updated_inputs, + outputs=updated_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + return updated_node_ports + + node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=auto_update, + ) + + assert node_ports.internal_inputs == original_inputs + assert node_ports.internal_outputs == original_outputs + + # this triggers an auto_update if auto_update is True + node_inputs = await node_ports.inputs + assert node_inputs == updated_inputs if auto_update else original_inputs + node_outputs = await node_ports.outputs + assert node_outputs == updated_outputs if auto_update else original_outputs + + +async def test_node_ports_accessors( + mock_db_manager: Callable, + default_configuration: Dict[str, Any], + node_uuid: str, +): + db_manager = mock_db_manager(default_configuration) + + original_inputs = create_valid_port_mapping(InputsList, suffix="original") + original_outputs = create_valid_port_mapping(OutputsList, suffix="original") + + async def mock_save_db_cb(*args, **kwargs): + pass + + async def mock_node_port_creator_cb(*args, **kwargs): + updated_node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + return updated_node_ports + + node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + + for port in original_inputs.values(): + assert await node_ports.get(port.key) == port.value + await node_ports.set(port.key, port.value) + + with pytest.raises(UnboundPortError): + await node_ports.get("some_invalid_key") + + for port in original_outputs.values(): + assert await node_ports.get(port.key) == port.value + await node_ports.set(port.key, port.value) + + +@pytest.fixture +async def mock_upload_file(mocker): + mock = mocker.patch( + "simcore_sdk.node_ports.filemanager.upload_file", + return_value=Future(), + ) + mock.return_value.set_result("0") + yield mock + + +async def test_node_ports_set_file_by_keymap( + mock_db_manager: Callable, + default_configuration: Dict[str, Any], + node_uuid: str, + mock_upload_file, +): + db_manager = mock_db_manager(default_configuration) + + original_inputs = create_valid_port_mapping(InputsList, suffix="original") + original_outputs = create_valid_port_mapping( + OutputsList, suffix="original", file_to_key=Path(__file__).name + ) + + async def mock_save_db_cb(*args, **kwargs): + pass + + async def mock_node_port_creator_cb(*args, **kwargs): + updated_node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + return updated_node_ports + + node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + + await node_ports.set_file_by_keymap(Path(__file__)) + + with pytest.raises(PortNotFound): + await node_ports.set_file_by_keymap(Path("/whatever/file/that/is/invalid")) + + +async def test_node_ports_v2_packages( + mock_db_manager: Callable, default_configuration: Dict[str, Any] +): + db_manager = mock_db_manager(default_configuration) + node_ports = await ports() + node_ports = await ports(db_manager) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index e14c4c349850..ed136b61df4b 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -19,6 +19,7 @@ from simcore_sdk.node_ports.exceptions import InvalidItemTypeError from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink from simcore_sdk.node_ports_v2.port import Port +from utils_port_v2 import create_valid_port_config from yarl import URL @@ -34,18 +35,6 @@ def camel_to_snake(name): ) -def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: - valid_config = { - "key": f"some_{conf_type}", - "label": "some label", - "description": "some description", - "type": conf_type, - "displayOrder": 2.3, - } - valid_config.update(kwargs) - return valid_config - - def this_node_file_name() -> Path: return Path(tempfile.gettempdir(), "this_node_file.txt") diff --git a/packages/simcore-sdk/tests/unit/test_port_mapping.py b/packages/simcore-sdk/tests/unit/test_port_mapping.py index d1e64dbc0e2a..915f1e07ba72 100644 --- a/packages/simcore-sdk/tests/unit/test_port_mapping.py +++ b/packages/simcore-sdk/tests/unit/test_port_mapping.py @@ -2,21 +2,8 @@ import pytest from simcore_sdk.node_ports.exceptions import UnboundPortError -from simcore_sdk.node_ports_v2.port import Port from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList - - -##################### HELPERS -def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: - valid_config = { - "key": f"some_{conf_type}", - "label": "some label", - "description": "some description", - "type": conf_type, - "displayOrder": 2.3, - } - valid_config.update(kwargs) - return valid_config +from utils_port_v2 import create_valid_port_config ##################### TESTS diff --git a/packages/simcore-sdk/tests/unit/test_serialization_v2.py b/packages/simcore-sdk/tests/unit/test_serialization_v2.py index a3bf84fcdb21..7cdbb53578e5 100644 --- a/packages/simcore-sdk/tests/unit/test_serialization_v2.py +++ b/packages/simcore-sdk/tests/unit/test_serialization_v2.py @@ -2,10 +2,7 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import asyncio -import json from typing import Any, Dict -from uuid import uuid4 import pytest from simcore_sdk.node_ports.dbmanager import DBManager @@ -14,42 +11,6 @@ create_nodeports_from_db, save_nodeports_to_db, ) -from sqlalchemy.exc import InvalidatePoolError - - -@pytest.fixture(scope="module") -def node_uuid() -> str: - return str(uuid4()) - - -@pytest.fixture(scope="function") -async def mock_db_manager( - loop: asyncio.AbstractEventLoop, - monkeypatch, - node_uuid: str, -): - def _mock_db_manager(port_cfg: Dict[str, Any]): - async def mock_get_ports_configuration_from_node_uuid(*args, **kwargs) -> str: - return json.dumps(port_cfg) - - async def mock_write_ports_configuration( - self, json_configuration: str, uuid: str - ): - assert json.loads(json_configuration) == port_cfg - assert uuid == node_uuid - - monkeypatch.setattr( - DBManager, - "get_ports_configuration_from_node_uuid", - mock_get_ports_configuration_from_node_uuid, - ) - monkeypatch.setattr( - DBManager, - "write_ports_configuration", - mock_write_ports_configuration, - ) - - yield _mock_db_manager @pytest.mark.parametrize("auto_update", [True, False]) @@ -59,8 +20,7 @@ async def test_create_nodeports_from_db( node_uuid: str, default_configuration: Dict[str, Any], ): - mock_db_manager(default_configuration) - db_manager = DBManager() + db_manager: DBManager = mock_db_manager(default_configuration) node_ports = await create_nodeports_from_db(db_manager, node_uuid, auto_update) assert node_ports.db_manager == db_manager assert node_ports.node_uuid == node_uuid @@ -75,10 +35,9 @@ async def test_create_nodeports_from_db_with_invalid_cfg( node_uuid: str, ): invalid_config = {"bad_key": "bad_value"} - mock_db_manager(invalid_config) - db_manager = DBManager() + db_manager: DBManager = mock_db_manager(invalid_config) with pytest.raises(InvalidProtocolError): - node_ports = await create_nodeports_from_db(db_manager, node_uuid) + _ = await create_nodeports_from_db(db_manager, node_uuid) async def test_save_nodeports_to_db( @@ -86,8 +45,7 @@ async def test_save_nodeports_to_db( node_uuid: str, default_configuration: Dict[str, Any], ): - mock_db_manager(default_configuration) - db_manager = DBManager() + db_manager: DBManager = mock_db_manager(default_configuration) node_ports = await create_nodeports_from_db(db_manager, node_uuid) await save_nodeports_to_db(node_ports) From 44a9f42b0328d6e061b054809e5eed650f31fea4 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 18:23:15 +0100 Subject: [PATCH 43/74] cleanup --- .../simcore-sdk/src/simcore_sdk/node_ports/filemanager.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py index c1b5ab895316..167249ca40d6 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py @@ -142,14 +142,6 @@ async def _download_link_to_file(session: ClientSession, url: URL, file_path: Pa return await response.release() -async def _file_sender(file_path: Path): - async with aiofiles.open(file_path, "rb") as file_pointer: - chunk = await file_pointer.read(CHUNK_SIZE) - while chunk: - yield chunk - chunk = await file_pointer.read(CHUNK_SIZE) - - async def _upload_file_to_link(session: ClientSession, url: URL, file_path: Path): log.debug("Uploading from %s to %s", file_path, url) async with session.put(url, data=file_path.open("rb")) as resp: From cd87fb27eba1abefc0337ba3bfb7a38cd19e2837 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 18:32:00 +0100 Subject: [PATCH 44/74] remove unused version --- .../simcore-sdk/src/simcore_sdk/node_ports/config.py | 1 - .../src/simcore_sdk/node_ports/exceptions.py | 11 ----------- .../src/simcore_sdk/node_ports/nodeports.py | 12 ++---------- .../src/simcore_sdk/node_ports/serialization.py | 2 -- 4 files changed, 2 insertions(+), 24 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py index f767bd2d77fa..cc0421410846 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py @@ -22,7 +22,6 @@ # ------------------------------------------------------------------------- NODE_KEYS: Dict[str, bool] = { - "version": True, "schema": True, "inputs": True, "outputs": True, diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py index 230fa3f015c3..288ed2a3c6ba 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py @@ -18,17 +18,6 @@ def __init__(self, obj): self.obj = obj -class WrongProtocolVersionError(NodeportsException): - """Using wrong protocol version""" - - def __init__(self, expected_version, found_version): - super().__init__( - f"Expecting version {expected_version}, found version {found_version}" - ) - self.expected_version = expected_version - self.found_version = found_version - - class UnboundPortError(NodeportsException, IndexError): """Accessed port is not configured""" diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py index 56ad267dafea..1710923d1b06 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py @@ -21,11 +21,8 @@ class Nodeports: """Allows the client to access the inputs and outputs assigned to the node""" - _version = "0.1" - def __init__( self, - version: str, input_schemas: SchemaItemsList = None, output_schemas: SchemaItemsList = None, input_payloads: DataItemsList = None, @@ -33,14 +30,10 @@ def __init__( ): log.debug( - "Initialising Nodeports object with version %s, inputs %s and outputs %s", - version, + "Initialising Nodeports object with inputs %s and outputs %s", input_payloads, outputs_payloads, ) - if self._version != version: - raise exceptions.WrongProtocolVersionError(self._version, version) - if not input_schemas: input_schemas = SchemaItemsList() if not output_schemas: @@ -59,8 +52,7 @@ def __init__( self.autowrite = False log.debug( - "Initialised Nodeports object with version %s, inputs %s and outputs %s", - version, + "Initialised Nodeports object with inputs %s and outputs %s", input_payloads, outputs_payloads, ) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py index a45cbbf16b55..1624d7db7816 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py @@ -93,7 +93,6 @@ def default(self, o): # pylint: disable=E0202 log.debug("Encoding Nodeports object") return { # pylint: disable=W0212 - "version": o._version, "schema": { "inputs": o._input_schemas, "outputs": o._output_schemas, @@ -142,7 +141,6 @@ def __decodeNodePorts(dct: Dict): ) return nodeports.Nodeports( - dct["version"], SchemaItemsList(decoded_input_schema), SchemaItemsList(decoded_output_schema), DataItemsList(decoded_input_payload), From 3fcb3cbe40cccf6bee06f344fb82d66fcdb56d2c Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 21:37:02 +0100 Subject: [PATCH 45/74] added standard package testing, linter --- .../src/simcore_sdk/node_ports/exceptions.py | 2 - packages/simcore-sdk/tests/conftest.py | 8 ++++ .../simcore-sdk/tests/unit/test_package.py | 38 +++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 packages/simcore-sdk/tests/unit/test_package.py diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py index 288ed2a3c6ba..318a135e1d8b 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py @@ -1,7 +1,5 @@ from typing import Optional -"""Defines the different exceptions that may arise in the nodeports package""" - class NodeportsException(Exception): """Basic exception for errors raised in nodeports""" diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index ec0f1bcdf91d..bc2461990ec6 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -7,6 +7,7 @@ from typing import Any, Dict import pytest +import simcore_sdk from simcore_sdk.node_ports import config as node_config ## HELPERS @@ -25,6 +26,13 @@ ] +@pytest.fixture(scope="session") +def package_dir(): + pdir = Path(simcore_sdk.__file__).resolve().parent + assert pdir.exists() + return pdir + + @pytest.fixture(scope="session") def osparc_simcore_root_dir() -> Path: """ osparc-simcore repo root dir """ diff --git a/packages/simcore-sdk/tests/unit/test_package.py b/packages/simcore-sdk/tests/unit/test_package.py new file mode 100644 index 000000000000..5b5a4338d3b9 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_package.py @@ -0,0 +1,38 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import os +import re +from pathlib import Path + +import pytest +from pytest_simcore.helpers.utils_pylint import assert_pylint_is_passing + + +@pytest.fixture +def pylintrc(osparc_simcore_root_dir): + pylintrc = osparc_simcore_root_dir / ".pylintrc" + assert pylintrc.exists() + return pylintrc + + +def test_run_pylint(pylintrc, package_dir): + assert_pylint_is_passing(pylintrc=pylintrc, package_dir=package_dir) + + +def test_no_pdbs_in_place(package_dir): + # TODO: add also test_dir excluding this function!? + # TODO: it can be commented! + # TODO: add check on other undesired code strings?! + MATCH = re.compile(r"pdb.set_trace()") + EXCLUDE = ["__pycache__", ".git"] + for root, dirs, files in os.walk(package_dir): + for name in files: + if name.endswith(".py"): + pypth = Path(root) / name + code = pypth.read_text() + found = MATCH.findall(code) + # TODO: should return line number + assert not found, "pbd.set_trace found in %s" % pypth + dirs[:] = [d for d in dirs if d not in EXCLUDE] From e02039cf2a83f5550717e2b787c3f32d467c0772 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 21:37:33 +0100 Subject: [PATCH 46/74] linter --- .../src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py | 1 - packages/simcore-sdk/tests/integration/np_helpers.py | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py index d6a3e1bc06c5..e36b9195920b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py +++ b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py @@ -1,5 +1,4 @@ import re -from unittest.mock import call import pytest from aioresponses import aioresponses diff --git a/packages/simcore-sdk/tests/integration/np_helpers.py b/packages/simcore-sdk/tests/integration/np_helpers.py index 170713ba79a4..343e8e180e81 100644 --- a/packages/simcore-sdk/tests/integration/np_helpers.py +++ b/packages/simcore-sdk/tests/integration/np_helpers.py @@ -3,7 +3,6 @@ # pylint:disable=redefined-outer-name # pylint:disable=too-many-arguments -import json import logging from pathlib import Path from typing import Dict From c2c7b47255b741a5a6f7cc4765b74e3148bd318c Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 22:08:38 +0100 Subject: [PATCH 47/74] mypy --- .../simcore_sdk/node_ports_v2/nodeports_v2.py | 10 +++---- .../src/simcore_sdk/node_ports_v2/port.py | 26 +++++++++---------- .../simcore_sdk/node_ports_v2/port_utils.py | 12 +++++---- .../node_ports_v2/ports_mapping.py | 14 +++++----- .../node_ports_v2/serialization_v2.py | 9 ++++--- 5 files changed, 38 insertions(+), 33 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 6ef2c5af481f..4b77549ae00d 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -6,7 +6,7 @@ from ..node_ports.dbmanager import DBManager from ..node_ports.exceptions import PortNotFound, UnboundPortError -from .port import ItemConcreteValue +from .links import ItemConcreteValue from .port_utils import is_file_type from .ports_mapping import InputsList, OutputsList @@ -25,7 +25,7 @@ class Nodeports(BaseModel): class Config: arbitrary_types_allowed = True - def __init__(self, **data): + def __init__(self, **data: Any): super().__init__(**data) # let's pass ourselves down for input_key in self.internal_inputs: @@ -56,7 +56,7 @@ async def get(self, item_key: str) -> ItemConcreteValue: # if this fails it will raise an exception return await (await self.outputs)[item_key].get() - async def set(self, item_key: str, item_value): + async def set(self, item_key: str, item_value: ItemConcreteValue) -> None: try: await (await self.inputs)[item_key].set(item_value) return @@ -66,7 +66,7 @@ async def set(self, item_key: str, item_value): # if this fails it will raise an exception return await (await self.outputs)[item_key].set(item_value) - async def set_file_by_keymap(self, item_value: Path): + async def set_file_by_keymap(self, item_value: Path) -> None: for output in (await self.outputs).values(): if is_file_type(output.property_type) and output.file_to_key_map: if item_value.name in output.file_to_key_map: @@ -74,7 +74,7 @@ async def set_file_by_keymap(self, item_value: Path): return raise PortNotFound(msg=f"output port for item {item_value} not found") - async def _node_ports_creator_cb(self, node_uuid: str) -> Type["NodePorts"]: + async def _node_ports_creator_cb(self, node_uuid: str) -> Type["Nodeports"]: return await self.node_port_creator_cb(self.db_manager, node_uuid) async def _auto_update_from_db(self) -> None: diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 41881216eb75..a7d25467974d 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -1,7 +1,7 @@ import logging from pathlib import Path from pprint import pformat -from typing import Dict, Optional, Tuple, Type +from typing import Any, Dict, Optional, Tuple, Type from models_library.services import PROPERTY_KEY_RE, ServiceProperty from pydantic import Field, PrivateAttr, validator @@ -23,7 +23,7 @@ class Port(ServiceProperty): key: str = Field(..., regex=PROPERTY_KEY_RE) - widget: Optional[Dict] = None + widget: Optional[Dict[str, Any]] = None value: Optional[DataItemValue] @@ -34,7 +34,7 @@ class Port(ServiceProperty): @validator("value", always=True) @classmethod - def ensure_value(cls, v, values): + def ensure_value(cls, v: DataItemValue, values: Dict[str, Any]) -> DataItemValue: if "property_type" in values and port_utils.is_file_type( values["property_type"] ): @@ -44,7 +44,7 @@ def ensure_value(cls, v, values): ) return v - def __init__(self, **data): + def __init__(self, **data: Any): super().__init__(**data) self._py_value_type = ( @@ -102,22 +102,22 @@ async def get(self) -> ItemConcreteValue: return self._py_value_converter(value) - async def set(self, new_value: ItemConcreteValue): + async def set(self, new_value: ItemConcreteValue) -> None: log.debug( "setting %s[%s] with value %s", self.key, self.property_type, new_value ) - converted_value = None + final_value: Optional[DataItemValue] = None if new_value is not None: # convert the concrete value to a data value - converted_value = self._py_value_converter(new_value) + converted_value: ItemConcreteValue = self._py_value_converter(new_value) - if port_utils.is_file_type(self.property_type): + if isinstance(converted_value, Path): if not converted_value.exists() or not converted_value.is_file(): - raise InvalidItemTypeError(self.property_type, new_value) - converted_value: FileLink = await port_utils.push_file_to_store( - converted_value - ) + raise InvalidItemTypeError(self.property_type, str(new_value)) + final_value = await port_utils.push_file_to_store(converted_value) + else: + final_value = converted_value - self.value = converted_value + self.value = final_value self._used_default_value = False await self._node_ports.save_to_db_cb(self._node_ports) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py index 6b1a02a0eaf0..62ff14a649ef 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -1,7 +1,7 @@ import logging import shutil from pathlib import Path -from typing import Any, Callable, Coroutine, Dict +from typing import Any, Callable, Coroutine, Dict, Optional from ..node_ports import config, data_items_utils, filemanager from .links import DownloadLink, FileLink, ItemConcreteValue, PortLink @@ -12,7 +12,7 @@ async def get_value_from_link( key: str, value: PortLink, - fileToKeyMap: Dict, + fileToKeyMap: Optional[Dict[str, str]], node_port_creator: Callable[[str], Coroutine[Any, Any, Any]], ) -> ItemConcreteValue: log.debug("Getting value %s", value) @@ -42,7 +42,9 @@ async def get_value_from_link( return value -async def pull_file_from_store(key: str, fileToKeyMap: Dict, value: FileLink) -> Path: +async def pull_file_from_store( + key: str, fileToKeyMap: Optional[Dict[str, str]], value: FileLink +) -> Path: log.debug("Getting value from storage %s", value) # do not make any assumption about s3_path, it is a str containing stuff that can be anything depending on the store local_path = data_items_utils.create_folder_path(key) @@ -74,7 +76,7 @@ async def push_file_to_store(file: Path) -> FileLink: async def pull_file_from_download_link( - key: str, fileToKeyMap: Dict, value: DownloadLink + key: str, fileToKeyMap: Optional[Dict[str, str]], value: DownloadLink ) -> Path: log.debug( "Getting value from download link [%s] with label %s", @@ -100,5 +102,5 @@ async def pull_file_from_download_link( return downloaded_file -def is_file_type(port_type: str): +def is_file_type(port_type: str) -> bool: return port_type.startswith("data:") diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py index 9ffa64760686..2afa839e4aaa 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py @@ -1,4 +1,4 @@ -from typing import Dict, Union +from typing import Dict, ItemsView, Iterator, KeysView, Union, ValuesView from models_library.services import PROPERTY_KEY_RE from pydantic import BaseModel, constr @@ -6,7 +6,7 @@ from ..node_ports.exceptions import UnboundPortError from .port import Port -PortKey = constr(regex=PROPERTY_KEY_RE) +PortKey: constr = constr(regex=PROPERTY_KEY_RE) class PortsMapping(BaseModel): @@ -20,19 +20,19 @@ def __getitem__(self, key: Union[int, PortKey]) -> Port: raise UnboundPortError(key) return self.__root__[key] - def __iter__(self): + def __iter__(self) -> Iterator[PortKey]: return iter(self.__root__) - def keys(self): + def keys(self) -> KeysView[PortKey]: return self.__root__.keys() - def items(self): + def items(self) -> ItemsView[PortKey, Port]: return self.__root__.items() - def values(self): + def values(self) -> ValuesView[Port]: return self.__root__.values() - def __len__(self): + def __len__(self) -> int: return self.__root__.__len__() diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py index 8748a4fd31d0..a7bfe42c5707 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -1,5 +1,5 @@ import json -from typing import List +from typing import Any, Dict, Set from aiopg.sa.result import RowProxy @@ -7,7 +7,7 @@ from ..node_ports.exceptions import InvalidProtocolError from .nodeports_v2 import Nodeports -NODE_REQUIRED_KEYS: List[str] = { +NODE_REQUIRED_KEYS: Set[str] = { "schema", "inputs", "outputs", @@ -26,7 +26,10 @@ async def create_nodeports_from_db( ) # convert to our internal node ports _PY_INT = "__root__" - node_ports_cfg = {"inputs": {_PY_INT: {}}, "outputs": {_PY_INT: {}}} + node_ports_cfg: Dict[str, Dict[str, Any]] = { + "inputs": {_PY_INT: {}}, + "outputs": {_PY_INT: {}}, + } for port_type in ["inputs", "outputs"]: # schemas first node_ports_cfg.update( From 8294c70a4f5e5e85ef277f0839fad121cd0e1bf5 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 22:26:59 +0100 Subject: [PATCH 48/74] improve testing --- packages/simcore-sdk/tests/unit/test_port.py | 26 ++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index ed136b61df4b..fdffabf3d4c4 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -70,16 +70,6 @@ def datcore_store_id() -> str: ##################### FIXTURES -@pytest.fixture -async def mock_upload_file(mocker): - mock = mocker.patch( - "simcore_sdk.node_ports.filemanager.upload_file", - return_value=Future(), - ) - mock.return_value.set_result(simcore_store_id()) - yield mock - - @pytest.fixture def this_node_file(tmp_path: Path) -> Path: file_path = this_node_file_name() @@ -157,6 +147,16 @@ async def mock_download_file_from_link( ) +@pytest.fixture +async def mock_upload_file(mocker): + mock = mocker.patch( + "simcore_sdk.node_ports.filemanager.upload_file", + return_value=Future(), + ) + mock.return_value.set_result(simcore_store_id()) + yield mock + + @pytest.fixture(autouse=True) def common_fixtures( loop, @@ -563,6 +563,11 @@ async def save_to_db_cb(self, node_ports): assert await port.get() == None else: assert await port.get() == exp_get_value + if isinstance(exp_value, PortLink) and isinstance(exp_get_value, Path): + # as the file is moved internally we need to re-create it or it fails + another_node_file_name().touch(exist_ok=True) + # it should work several times + assert await port.get() == exp_get_value # set a new value await port.set(new_value) @@ -576,6 +581,7 @@ async def save_to_db_cb(self, node_ports): assert await port.get() == None else: assert await port.get() == exp_new_get_value + assert await port.get() == exp_new_get_value @pytest.mark.parametrize( From a5f9434cefd8e2908a57eab06b85b4a1a6bd1ea3 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 7 Dec 2020 22:27:08 +0100 Subject: [PATCH 49/74] add ETag --- .../models-library/src/models_library/projects_nodes_io.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/models-library/src/models_library/projects_nodes_io.py b/packages/models-library/src/models_library/projects_nodes_io.py index 4836a2854955..1fd8f31828dc 100644 --- a/packages/models-library/src/models_library/projects_nodes_io.py +++ b/packages/models-library/src/models_library/projects_nodes_io.py @@ -60,6 +60,10 @@ class BaseFileLink(BaseModel): "94453a6a-c8d4-52b3-a22d-ccbf81f8d636/d4442ca4-23fd-5b6b-ba6d-0b75f711c109/y_1D.txt", ], ) + e_tag: Optional[str] = Field( + None, + description="Entity tag that uniquely represents the file. The method to generate the tag is not specified (black box).", + ) class Config: extra = Extra.forbid From c0500fe95fdb88ffab3e0727427ba6c0dee3617e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 09:21:34 +0100 Subject: [PATCH 50/74] added ETag when uploading file --- .../src/simcore_sdk/node_ports/_item.py | 2 +- .../src/simcore_sdk/node_ports/filemanager.py | 58 ++++++++++++++----- .../simcore_sdk/node_ports_v2/port_utils.py | 6 +- .../tests/integration/test_filemanager.py | 9 +-- .../tests/unit/test_nodeports_v2.py | 9 ++- packages/simcore-sdk/tests/unit/test_port.py | 20 ++++++- 6 files changed, 79 insertions(+), 25 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py index 0119508e795c..5b26338f6c7d 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py @@ -161,7 +161,7 @@ async def set(self, value: ItemConcreteValue): s3_object = data_items_utils.encode_file_id( file_path, project_id=config.PROJECT_ID, node_id=config.NODE_UUID ) - store_id = await filemanager.upload_file( + store_id, _ = await filemanager.upload_file( store_name=config.STORE, s3_object=s3_object, local_file_path=file_path ) log.debug("file path %s uploaded", value) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py index 167249ca40d6..b93508d0808d 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py @@ -3,7 +3,7 @@ import warnings from contextlib import contextmanager from pathlib import Path -from typing import Optional +from typing import Any, Dict, Optional, Tuple import aiofiles from aiohttp import ClientSession, ClientTimeout @@ -124,6 +124,24 @@ async def _get_upload_link(store_id: int, file_id: str, api: UsersApi) -> URL: return await _get_link(store_id, file_id, api.upload_file) +async def _get_file_metadata(store_id: int, file_id: str) -> Dict[str, Any]: + log.debug("Getting file metadata from store id [%s] for %s", store_id, file_id) + try: + with api_client() as client: + api = UsersApi(client) + resp = await api.get_file_metadata( + file_id=file_id, location_id=store_id, user_id=config.USER_ID + ) + if resp.error: + raise exceptions.StorageServerIssue( + f"getting file {file_id} metadata failed: [{resp.error.to_str()}]" + ) + return resp.data + + except ApiException as err: + _handle_api_exception(store_id, err) + + async def _download_link_to_file(session: ClientSession, url: URL, file_path: Path): log.debug("Downloading from %s to %s", url, file_path) async with session.get(url) as response: @@ -142,7 +160,12 @@ async def _download_link_to_file(session: ClientSession, url: URL, file_path: Pa return await response.release() -async def _upload_file_to_link(session: ClientSession, url: URL, file_path: Path): +ETag = str + + +async def _upload_file_to_link( + session: ClientSession, url: URL, file_path: Path +) -> Optional[ETag]: log.debug("Uploading from %s to %s", file_path, url) async with session.put(url, data=file_path.open("rb")) as resp: if resp.status > 299: @@ -150,6 +173,15 @@ async def _upload_file_to_link(session: ClientSession, url: URL, file_path: Path raise exceptions.S3TransferError( "Could not upload file {}:{}".format(file_path, response_text) ) + if resp.status != 200: + response_text = await resp.text() + raise exceptions.S3TransferError( + "Issue when uploading file {}:{}".format(file_path, response_text) + ) + # get the S3 etag from the headers + e_tag = resp.headers.get("Etag", None) + log.debug("Uploaded %s to %s, received Etag %s", file_path, url, e_tag) + return e_tag async def download_file_from_s3( @@ -158,7 +190,7 @@ async def download_file_from_s3( store_id: str = None, s3_object: str, local_folder: Path, - session: Optional[ClientSession] = None + session: Optional[ClientSession] = None, ) -> Path: """Downloads a file from S3 @@ -215,12 +247,12 @@ async def download_file_from_link( async def upload_file( *, - store_id: str = None, - store_name: str = None, + store_id: Optional[str] = None, + store_name: Optional[str] = None, s3_object: str, local_file_path: Path, - session: Optional[ClientSession] = None -) -> str: + session: Optional[ClientSession] = None, +) -> Tuple[str, str]: """Uploads a file to S3 :param session: add app[APP_CLIENT_SESSION_KEY] session here otherwise default is opened/closed every call @@ -243,14 +275,14 @@ async def upload_file( if store_name is not None: store_id = await _get_location_id_from_location_name(store_name, api) - upload_link = await _get_upload_link(store_id, s3_object, api) + upload_link: URL = await _get_upload_link(store_id, s3_object, api) if upload_link: - upload_link = URL(upload_link) - + # FIXME: This client should be kept with the nodeports instead of creating one each time async with ClientSessionContextManager(session) as active_session: - await _upload_file_to_link(active_session, upload_link, local_file_path) - - return store_id + e_tag = await _upload_file_to_link( + active_session, upload_link, local_file_path + ) + return store_id, e_tag raise exceptions.S3InvalidPathError(s3_object) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py index 62ff14a649ef..02150bfd2ab8 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -68,11 +68,11 @@ async def push_file_to_store(file: Path) -> FileLink: s3_object = data_items_utils.encode_file_id( file, project_id=config.PROJECT_ID, node_id=config.NODE_UUID ) - store_id = await filemanager.upload_file( + store_id, e_tag = await filemanager.upload_file( store_name=config.STORE, s3_object=s3_object, local_file_path=file ) - log.debug("file path %s uploaded", file) - return FileLink(store=store_id, path=s3_object) + log.debug("file path %s uploaded, received ETag %s", file, e_tag) + return FileLink(store=store_id, path=s3_object, e_tag=e_tag) async def pull_file_from_download_link( diff --git a/packages/simcore-sdk/tests/integration/test_filemanager.py b/packages/simcore-sdk/tests/integration/test_filemanager.py index 3955f130f8aa..91e5b9474908 100644 --- a/packages/simcore-sdk/tests/integration/test_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_filemanager.py @@ -27,14 +27,15 @@ async def test_valid_upload_download( assert file_path.exists() file_id = file_uuid(file_path) - store = s3_simcore_location - await filemanager.upload_file( - store_id=store, s3_object=file_id, local_file_path=file_path + store_id, e_tag = await filemanager.upload_file( + store_id=s3_simcore_location, s3_object=file_id, local_file_path=file_path ) + assert store_id == s3_simcore_location + assert e_tag download_folder = Path(tmpdir) / "downloads" download_file_path = await filemanager.download_file_from_s3( - store_id=store, s3_object=file_id, local_folder=download_folder + store_id=s3_simcore_location, s3_object=file_id, local_folder=download_folder ) assert download_file_path.exists() assert download_file_path.name == "test.test" diff --git a/packages/simcore-sdk/tests/unit/test_nodeports_v2.py b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py index 8e07756aec23..233792a61b69 100644 --- a/packages/simcore-sdk/tests/unit/test_nodeports_v2.py +++ b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py @@ -116,13 +116,18 @@ async def mock_node_port_creator_cb(*args, **kwargs): await node_ports.set(port.key, port.value) +@pytest.fixture(scope="session") +def e_tag() -> str: + return "123154654684321-1" + + @pytest.fixture -async def mock_upload_file(mocker): +async def mock_upload_file(mocker, e_tag): mock = mocker.patch( "simcore_sdk.node_ports.filemanager.upload_file", return_value=Future(), ) - mock.return_value.set_result("0") + mock.return_value.set_result(("0", e_tag)) yield mock diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index fdffabf3d4c4..ac4e8dafb113 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -67,6 +67,10 @@ def datcore_store_id() -> str: return "1" +def e_tag() -> str: + return "1212132546546321-1" + + ##################### FIXTURES @@ -147,13 +151,18 @@ async def mock_download_file_from_link( ) +@pytest.fixture(scope="session", name="e_tag") +def e_tag_fixture() -> str: + return "1212132546546321-1" + + @pytest.fixture -async def mock_upload_file(mocker): +async def mock_upload_file(mocker, e_tag): mock = mocker.patch( "simcore_sdk.node_ports.filemanager.upload_file", return_value=Future(), ) - mock.return_value.set_result(simcore_store_id()) + mock.return_value.set_result((simcore_store_id(), e_tag)) yield mock @@ -247,6 +256,7 @@ def common_fixtures( exp_new_value=FileLink( store=simcore_store_id(), path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), ), exp_new_get_value=download_file_folder_name() / "no_file" @@ -269,6 +279,7 @@ def common_fixtures( exp_new_value=FileLink( store=simcore_store_id(), path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), ), exp_new_get_value=download_file_folder_name() / "no_file_with_default" @@ -356,6 +367,7 @@ def common_fixtures( exp_new_value=FileLink( store=simcore_store_id(), path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), ), exp_new_get_value=download_file_folder_name() / "some_file_on_datcore" @@ -384,6 +396,7 @@ def common_fixtures( exp_new_value=FileLink( store=simcore_store_id(), path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), ), exp_new_get_value=download_file_folder_name() / "download_link" @@ -415,6 +428,7 @@ def common_fixtures( exp_new_value=FileLink( store=simcore_store_id(), path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), ), exp_new_get_value=download_file_folder_name() / "download_link_with_file_to_key" @@ -445,6 +459,7 @@ def common_fixtures( exp_new_value=FileLink( store=simcore_store_id(), path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), ), exp_new_get_value=download_file_folder_name() / "file_port_link" @@ -478,6 +493,7 @@ def common_fixtures( exp_new_value=FileLink( store=simcore_store_id(), path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), ), exp_new_get_value=download_file_folder_name() / "file_port_link_with_file_to_key_map" From 64205687c8bf34a5fdcf9773e2acd0b06d7e8c50 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 09:38:17 +0100 Subject: [PATCH 51/74] fix test related to uploaded file and received eTag --- .../src/models_library/projects_nodes_io.py | 1 + .../tests/integration/test_nodeports2.py | 32 +++++++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/packages/models-library/src/models_library/projects_nodes_io.py b/packages/models-library/src/models_library/projects_nodes_io.py index 1fd8f31828dc..34bd937ea386 100644 --- a/packages/models-library/src/models_library/projects_nodes_io.py +++ b/packages/models-library/src/models_library/projects_nodes_io.py @@ -63,6 +63,7 @@ class BaseFileLink(BaseModel): e_tag: Optional[str] = Field( None, description="Entity tag that uniquely represents the file. The method to generate the tag is not specified (black box).", + alias="eTag", ) class Config: diff --git a/packages/simcore-sdk/tests/integration/test_nodeports2.py b/packages/simcore-sdk/tests/integration/test_nodeports2.py index 5a8d6fbf2c26..3014573c7aa0 100644 --- a/packages/simcore-sdk/tests/integration/test_nodeports2.py +++ b/packages/simcore-sdk/tests/integration/test_nodeports2.py @@ -90,6 +90,11 @@ async def check_config_valid(ports: Nodeports, config_dict: Dict): await _check_ports_valid(ports, config_dict, "outputs") +@pytest.fixture(scope="session") +def e_tag() -> str: + return "123154654684321-1" + + async def test_default_configuration( default_configuration: Dict, ): # pylint: disable=W0613, W0621 @@ -169,6 +174,7 @@ async def test_port_file_accessors( item_value: str, item_pytype: Type, config_value: Dict[str, str], + e_tag: str, ): # pylint: disable=W0613, W0621 config_dict, project_id, node_uuid = special_configuration( inputs=[("in_1", item_type, config_value)], @@ -183,12 +189,17 @@ async def test_port_file_accessors( # this triggers an upload to S3 + configuration change await (await PORTS.outputs)["out_34"].set(item_value) # this is the link to S3 storage - assert (await PORTS.outputs)["out_34"].value.dict( + received_file_link = (await PORTS.outputs)["out_34"].value.dict( by_alias=True, exclude_unset=True - ) == { - "store": s3_simcore_location, - "path": Path(str(project_id), str(node_uuid), Path(item_value).name).as_posix(), - } + ) + assert received_file_link["store"] == s3_simcore_location + assert ( + received_file_link["path"] + == Path(str(project_id), str(node_uuid), Path(item_value).name).as_posix() + ) + # the eTag is created by the S3 server + assert received_file_link["eTag"] + # this triggers a download from S3 to a location in /tempdir/simcorefiles/item_key assert isinstance(await (await PORTS.outputs)["out_34"].get(), item_pytype) assert (await (await PORTS.outputs)["out_34"].get()).exists() @@ -440,9 +451,10 @@ async def test_file_mapping( await PORTS.set_file_by_keymap(file_path) file_id = np_helpers.file_uuid(file_path, project_id, node_uuid) - assert (await PORTS.outputs)["out_1"].value.dict( + received_file_link = (await PORTS.outputs)["out_1"].value.dict( by_alias=True, exclude_unset=True - ) == { - "store": s3_simcore_location, - "path": file_id, - } + ) + assert received_file_link["store"] == s3_simcore_location + assert received_file_link["path"] == file_id + # received a new eTag + assert received_file_link["eTag"] From 160fd04f3f571996aec40234b83033c32078c536 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 13:46:18 +0100 Subject: [PATCH 52/74] cleaning up fixing small issues --- .../src/simcore_sdk/node_ports_v2/__init__.py | 3 ++- .../simcore_sdk/node_ports_v2/serialization_v2.py | 9 ++++++--- .../tests/integration/test_nodeports2.py | 2 +- .../simcore-sdk/tests/unit/test_nodeports_v2.py | 7 +++---- packages/simcore-sdk/tests/unit/test_port.py | 15 +++++++-------- .../simcore-sdk/tests/unit/test_port_mapping.py | 6 +++--- .../tests/unit/test_serialization_v2.py | 5 ++--- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py index 4ff38a59920c..cd96448bbd09 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py @@ -5,6 +5,7 @@ from ..node_ports import exceptions from ..node_ports.dbmanager import DBManager from .nodeports_v2 import Nodeports +from .port import Port from .serialization_v2 import create_nodeports_from_db # nodeports is a library for accessing data linked to the node @@ -23,4 +24,4 @@ async def ports(db_manager: Optional[DBManager] = None) -> Nodeports: ) -__all__ = ["ports", "node_config", "exceptions"] +__all__ = ["ports", "node_config", "exceptions", "Port"] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py index a7bfe42c5707..0580f369c27a 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -73,9 +73,12 @@ async def save_nodeports_to_db(nodeports: Nodeports) -> None: port_cfg["schema"][port_type][port_key] = key_schema # payload (only if default value was not used) # pylint: disable=protected-access - if not getattr(nodeports, f"internal_{port_type}")[ - port_key - ]._used_default_value: + if ( + port_values["value"] is not None + and not getattr(nodeports, f"internal_{port_type}")[ + port_key + ]._used_default_value + ): port_cfg[port_type][port_key] = port_values["value"] await nodeports.db_manager.write_ports_configuration( diff --git a/packages/simcore-sdk/tests/integration/test_nodeports2.py b/packages/simcore-sdk/tests/integration/test_nodeports2.py index 3014573c7aa0..b412ba0c3adf 100644 --- a/packages/simcore-sdk/tests/integration/test_nodeports2.py +++ b/packages/simcore-sdk/tests/integration/test_nodeports2.py @@ -13,7 +13,7 @@ import pytest import sqlalchemy as sa from simcore_sdk import node_ports_v2 -from simcore_sdk.node_ports import exceptions +from simcore_sdk.node_ports_v2 import exceptions from simcore_sdk.node_ports_v2.links import ItemConcreteValue from simcore_sdk.node_ports_v2.nodeports_v2 import Nodeports diff --git a/packages/simcore-sdk/tests/unit/test_nodeports_v2.py b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py index 233792a61b69..735f8bfc11d1 100644 --- a/packages/simcore-sdk/tests/unit/test_nodeports_v2.py +++ b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py @@ -7,8 +7,7 @@ from typing import Any, Callable, Dict import pytest -from simcore_sdk.node_ports.exceptions import PortNotFound, UnboundPortError -from simcore_sdk.node_ports_v2 import Nodeports, ports +from simcore_sdk.node_ports_v2 import Nodeports, exceptions, ports from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList from utils_port_v2 import create_valid_port_mapping @@ -108,7 +107,7 @@ async def mock_node_port_creator_cb(*args, **kwargs): assert await node_ports.get(port.key) == port.value await node_ports.set(port.key, port.value) - with pytest.raises(UnboundPortError): + with pytest.raises(exceptions.UnboundPortError): await node_ports.get("some_invalid_key") for port in original_outputs.values(): @@ -171,7 +170,7 @@ async def mock_node_port_creator_cb(*args, **kwargs): await node_ports.set_file_by_keymap(Path(__file__)) - with pytest.raises(PortNotFound): + with pytest.raises(exceptions.PortNotFound): await node_ports.set_file_by_keymap(Path("/whatever/file/that/is/invalid")) diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index ac4e8dafb113..9425c3f2cd44 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -15,8 +15,7 @@ import pytest from aiohttp.client import ClientSession from pydantic.error_wrappers import ValidationError -from simcore_sdk.node_ports import config -from simcore_sdk.node_ports.exceptions import InvalidItemTypeError +from simcore_sdk.node_ports_v2 import exceptions, node_config from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink from simcore_sdk.node_ports_v2.port import Port from utils_port_v2 import create_valid_port_config @@ -181,10 +180,10 @@ def common_fixtures( ): """this module main fixture""" - config.USER_ID = user_id - config.PROJECT_ID = project_id - config.NODE_UUID = node_uuid - config.STORAGE_ENDPOINT = "storage:8080" + node_config.USER_ID = user_id + node_config.PROJECT_ID = project_id + node_config.NODE_UUID = node_uuid + node_config.STORAGE_ENDPOINT = "storage:8080" ##################### TESTS @@ -645,9 +644,9 @@ def test_invalid_port(port_cfg: Dict[str, Any]): async def test_invalid_file_type_setter(port_cfg: Dict[str, Any]): port = Port(**port_cfg) # set a file that does not exist - with pytest.raises(InvalidItemTypeError): + with pytest.raises(exceptions.InvalidItemTypeError): await port.set("some/dummy/file/name") # set a folder fails too - with pytest.raises(InvalidItemTypeError): + with pytest.raises(exceptions.InvalidItemTypeError): await port.set(Path(__file__).parent) diff --git a/packages/simcore-sdk/tests/unit/test_port_mapping.py b/packages/simcore-sdk/tests/unit/test_port_mapping.py index 915f1e07ba72..e1395c376366 100644 --- a/packages/simcore-sdk/tests/unit/test_port_mapping.py +++ b/packages/simcore-sdk/tests/unit/test_port_mapping.py @@ -1,7 +1,7 @@ from typing import Any, Dict, Type, Union import pytest -from simcore_sdk.node_ports.exceptions import UnboundPortError +from simcore_sdk.node_ports_v2 import exceptions from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList from utils_port_v2 import create_valid_port_config @@ -39,8 +39,8 @@ def test_filled_ports_mapping(port_class: Type[Union[InputsList, OutputsList]]): for index, port_key in enumerate(port_cfgs): assert port_mapping[index] == port_mapping[port_key] - with pytest.raises(UnboundPortError): + with pytest.raises(exceptions.UnboundPortError): _ = port_mapping[len(port_cfgs)] - with pytest.raises(UnboundPortError): + with pytest.raises(exceptions.UnboundPortError): _ = port_mapping["whatever"] diff --git a/packages/simcore-sdk/tests/unit/test_serialization_v2.py b/packages/simcore-sdk/tests/unit/test_serialization_v2.py index 7cdbb53578e5..9833f5146cd1 100644 --- a/packages/simcore-sdk/tests/unit/test_serialization_v2.py +++ b/packages/simcore-sdk/tests/unit/test_serialization_v2.py @@ -5,8 +5,7 @@ from typing import Any, Dict import pytest -from simcore_sdk.node_ports.dbmanager import DBManager -from simcore_sdk.node_ports.exceptions import InvalidProtocolError +from simcore_sdk.node_ports_v2 import DBManager, exceptions from simcore_sdk.node_ports_v2.serialization_v2 import ( create_nodeports_from_db, save_nodeports_to_db, @@ -36,7 +35,7 @@ async def test_create_nodeports_from_db_with_invalid_cfg( ): invalid_config = {"bad_key": "bad_value"} db_manager: DBManager = mock_db_manager(invalid_config) - with pytest.raises(InvalidProtocolError): + with pytest.raises(exceptions.InvalidProtocolError): _ = await create_nodeports_from_db(db_manager, node_uuid) From f9105622bc7315bb3987c864c2190807aaddbb80 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 13:46:45 +0100 Subject: [PATCH 53/74] add eTag into project json schema --- .../schemas/project-v0.0.1-converted.yaml | 4 ++ api/specs/common/schemas/project-v0.0.1.json | 11 ++++- .../api/v0/schemas/project-v0.0.1.json | 11 ++++- .../api/v0/openapi.yaml | 16 ++++++++ .../api/v0/schemas/project-v0.0.1.json | 11 ++++- .../api/v0/openapi.yaml | 40 +++++++++++++++++++ .../api/v0/schemas/project-v0.0.1.json | 11 ++++- 7 files changed, 100 insertions(+), 4 deletions(-) diff --git a/api/specs/common/schemas/project-v0.0.1-converted.yaml b/api/specs/common/schemas/project-v0.0.1-converted.yaml index b4e0fbd30f4e..48c573471710 100644 --- a/api/specs/common/schemas/project-v0.0.1-converted.yaml +++ b/api/specs/common/schemas/project-v0.0.1-converted.yaml @@ -159,6 +159,8 @@ properties: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -221,6 +223,8 @@ properties: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: diff --git a/api/specs/common/schemas/project-v0.0.1.json b/api/specs/common/schemas/project-v0.0.1.json index d61d3cc2107c..a0ba4137bfdd 100644 --- a/api/specs/common/schemas/project-v0.0.1.json +++ b/api/specs/common/schemas/project-v0.0.1.json @@ -210,6 +210,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -302,6 +305,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -344,7 +350,10 @@ ] }, "parent": { - "type": [ "null", "string" ], + "type": [ + "null", + "string" + ], "format": "uuid", "description": "Parent's (group-nodes') node ID s.", "examples": [ diff --git a/services/director/src/simcore_service_director/api/v0/schemas/project-v0.0.1.json b/services/director/src/simcore_service_director/api/v0/schemas/project-v0.0.1.json index d61d3cc2107c..a0ba4137bfdd 100644 --- a/services/director/src/simcore_service_director/api/v0/schemas/project-v0.0.1.json +++ b/services/director/src/simcore_service_director/api/v0/schemas/project-v0.0.1.json @@ -210,6 +210,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -302,6 +305,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -344,7 +350,10 @@ ] }, "parent": { - "type": [ "null", "string" ], + "type": [ + "null", + "string" + ], "format": "uuid", "description": "Parent's (group-nodes') node ID s.", "examples": [ diff --git a/services/storage/src/simcore_service_storage/api/v0/openapi.yaml b/services/storage/src/simcore_service_storage/api/v0/openapi.yaml index 16572cd03967..279bc9db0a54 100644 --- a/services/storage/src/simcore_service_storage/api/v0/openapi.yaml +++ b/services/storage/src/simcore_service_storage/api/v0/openapi.yaml @@ -1886,6 +1886,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -1948,6 +1950,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -2315,6 +2319,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -2377,6 +2383,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -2754,6 +2762,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -2816,6 +2826,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -3293,6 +3305,8 @@ components: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -3355,6 +3369,8 @@ components: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: diff --git a/services/storage/src/simcore_service_storage/api/v0/schemas/project-v0.0.1.json b/services/storage/src/simcore_service_storage/api/v0/schemas/project-v0.0.1.json index d61d3cc2107c..a0ba4137bfdd 100644 --- a/services/storage/src/simcore_service_storage/api/v0/schemas/project-v0.0.1.json +++ b/services/storage/src/simcore_service_storage/api/v0/schemas/project-v0.0.1.json @@ -210,6 +210,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -302,6 +305,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -344,7 +350,10 @@ ] }, "parent": { - "type": [ "null", "string" ], + "type": [ + "null", + "string" + ], "format": "uuid", "description": "Parent's (group-nodes') node ID s.", "examples": [ diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 43f4078df06b..7e1f645bcecb 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -5926,6 +5926,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -5988,6 +5990,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -6486,6 +6490,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -6548,6 +6554,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -6926,6 +6934,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -6988,6 +6998,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -7484,6 +7496,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -7546,6 +7560,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -8048,6 +8064,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -8110,6 +8128,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -8603,6 +8623,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -8665,6 +8687,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -9043,6 +9067,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -9105,6 +9131,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -9623,6 +9651,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -9685,6 +9715,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -11015,6 +11047,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -11077,6 +11111,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -11572,6 +11608,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -11634,6 +11672,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: diff --git a/services/web/server/src/simcore_service_webserver/api/v0/schemas/project-v0.0.1.json b/services/web/server/src/simcore_service_webserver/api/v0/schemas/project-v0.0.1.json index d61d3cc2107c..a0ba4137bfdd 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/schemas/project-v0.0.1.json +++ b/services/web/server/src/simcore_service_webserver/api/v0/schemas/project-v0.0.1.json @@ -210,6 +210,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -302,6 +305,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -344,7 +350,10 @@ ] }, "parent": { - "type": [ "null", "string" ], + "type": [ + "null", + "string" + ], "format": "uuid", "description": "Parent's (group-nodes') node ID s.", "examples": [ From 32a65f2535145cc6441381dbcc4d80be55d16486 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 13:47:49 +0100 Subject: [PATCH 54/74] bonus: fix progress already in percent --- .../web/client/source/class/osparc/desktop/WorkbenchView.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index 6ea7a662a731..c1734a357cc0 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -632,7 +632,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { socket.on(slotName2, function(data) { const d = JSON.parse(data); const nodeId = d["Node"]; - const progress = 100 * Number.parseFloat(d["Progress"]).toFixed(4); + const progress = Number.parseFloat(d["Progress"]).toFixed(4); const workbench = this.getStudy().getWorkbench(); const node = workbench.getNode(nodeId); if (node) { From 04a82a138d20c71fd02b6e0b2ca0e27e081c77a2 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 13:48:02 +0100 Subject: [PATCH 55/74] switch sidecar to node ports v2 --- .../src/simcore_service_sidecar/core.py | 12 +-- .../src/simcore_service_sidecar/executor.py | 73 ++++++++++--------- .../src/simcore_service_sidecar/rabbitmq.py | 1 + .../sidecar/tests/integration/test_sidecar.py | 12 +-- 4 files changed, 50 insertions(+), 48 deletions(-) diff --git a/services/sidecar/src/simcore_service_sidecar/core.py b/services/sidecar/src/simcore_service_sidecar/core.py index ba03a26a5c8d..5cf2a2661945 100644 --- a/services/sidecar/src/simcore_service_sidecar/core.py +++ b/services/sidecar/src/simcore_service_sidecar/core.py @@ -12,8 +12,8 @@ comp_pipeline, comp_tasks, ) -from simcore_sdk import node_ports -from simcore_sdk.node_ports import log as node_port_log +from simcore_sdk import node_ports_v2 +from simcore_sdk.node_ports_v2 import log as node_port_v2_log from sqlalchemy import and_, literal_column from . import config, exceptions @@ -24,7 +24,7 @@ log = get_task_logger(__name__) log.setLevel(config.SIDECAR_LOGLEVEL) -node_port_log.setLevel(config.SIDECAR_LOGLEVEL) +node_port_v2_log.setLevel(config.SIDECAR_LOGLEVEL) async def task_required_resources(node_id: str) -> Union[Dict[str, bool], None]: @@ -220,9 +220,9 @@ async def inspect( ) # config nodeports - node_ports.node_config.USER_ID = user_id - node_ports.node_config.NODE_UUID = task.node_id - node_ports.node_config.PROJECT_ID = task.project_id + node_ports_v2.node_config.USER_ID = user_id + node_ports_v2.node_config.NODE_UUID = task.node_id + node_ports_v2.node_config.PROJECT_ID = task.project_id # now proceed actually running the task (we do that after the db session has been closed) # try to run the task, return empyt list of next nodes if anything goes wrong diff --git a/services/sidecar/src/simcore_service_sidecar/executor.py b/services/sidecar/src/simcore_service_sidecar/executor.py index a766c2d8236a..ccc6ceaa8aca 100644 --- a/services/sidecar/src/simcore_service_sidecar/executor.py +++ b/services/sidecar/src/simcore_service_sidecar/executor.py @@ -16,8 +16,8 @@ from celery.utils.log import get_task_logger from packaging import version from servicelib.utils import fire_and_forget_task, logged_gather -from simcore_sdk import node_data, node_ports -from simcore_sdk.node_ports.dbmanager import DBManager +from simcore_sdk import node_data, node_ports_v2 +from simcore_sdk.node_ports_v2 import DBManager from . import config, exceptions from .boot_mode import get_boot_mode @@ -146,37 +146,36 @@ async def _get_node_ports(self): if self.db_manager is None: # Keeps single db engine: simcore_sdk.node_ports.dbmanager_{id} self.db_manager = DBManager(self.db_engine) - return await node_ports.ports(self.db_manager) + return await node_ports_v2.ports(self.db_manager) - async def _process_task_input(self, port: node_ports.Port, input_ports: Dict): + async def _process_task_input(self, port: node_ports_v2.Port, input_ports: Dict): + log.debug("getting value from node ports...") port_value = await port.get() - log.debug("PROCESSING %s %s:%s", port.key, type(port_value), port_value) - if str(port.type).startswith("data:"): - path = port_value - if path: - # the filename is not necessarily the name of the port, might be mapped - mapped_filename = Path(path).name - input_ports[port.key] = str(port_value) - final_path = self.shared_folders.input_folder / mapped_filename - shutil.copy(str(path), str(final_path)) - log.debug( - "DOWNLOAD successfull from %s to %s via %s", - port.key, - final_path, - path, - ) - # check if the file is a zip, in that case extract all if the service does not expect a zip file - if zipfile.is_zipfile(final_path) and ( - str(port.type) != "data:application/zip" - ): - with zipfile.ZipFile(final_path, "r") as zip_obj: - zip_obj.extractall(final_path.parents[0]) - # finally remove the zip archive - os.remove(final_path) - else: - input_ports[port.key] = port_value - else: - input_ports[port.key] = port_value + input_ports[port.key] = port_value + log.debug("PROCESSING %s [%s]: %s", port.key, type(port_value), port_value) + if port_value is None: + # we are done here + return + + if isinstance(port_value, Path): + input_ports[port.key] = str(port_value) + # treat files specially, the file shall be moved to the right location + final_path = self.shared_folders.input_folder / port_value.name + shutil.move(port_value, final_path) + log.debug( + "DOWNLOAD successfull from %s to %s via %s", + port.key, + final_path, + port_value, + ) + # check if the file is a zip, in that case extract all if the service does not expect a zip file + if zipfile.is_zipfile(final_path) and ( + str(port.type) != "data:application/zip" + ): + with zipfile.ZipFile(final_path, "r") as zip_obj: + zip_obj.extractall(final_path.parents[0]) + # finally remove the zip archive + os.remove(final_path) async def _process_task_inputs(self) -> Dict: log.debug("Inputs parsing...") @@ -184,7 +183,7 @@ async def _process_task_inputs(self) -> Dict: input_ports: Dict = {} try: PORTS = await self._get_node_ports() - except node_ports.exceptions.NodeNotFound: + except node_ports_v2.exceptions.NodeNotFound: await self._error_message_to_ui_and_logs( "Missing node information in the database" ) @@ -197,7 +196,7 @@ async def _process_task_inputs(self) -> Dict: await logged_gather( *[ self._process_task_input(port, input_ports) - for port in (await PORTS.inputs) + for port in (await PORTS.inputs).values() ] ) log.debug("Inputs parsing DONE") @@ -324,6 +323,8 @@ async def _run_container(self): LogType.LOG, f"[sidecar]Running {self.task.image['name']}:{self.task.image['tag']}...", ) + # ensure progress 0.0 is sent + await self._post_messages(LogType.PROGRESS, "0.0") container = await docker_client.containers.create( config=container_config ) @@ -447,7 +448,7 @@ async def _process_task_output(self): with file_path.open() as fp: output_ports = json.load(fp) task_outputs = await PORTS.outputs - for port in task_outputs: + for port in task_outputs.values(): if port.key in output_ports.keys(): await port.set(output_ports[port.key]) else: @@ -458,7 +459,7 @@ async def _process_task_output(self): # WARNING: nodeports is NOT concurrent-safe, dont' use gather here for coro in file_upload_tasks: await coro - except node_ports.exceptions.NodeNotFound: + except node_ports_v2.exceptions.NodeNotFound: await self._error_message_to_ui_and_logs( "Error: no ports info found in the database." ) @@ -466,7 +467,7 @@ async def _process_task_output(self): await self._error_message_to_ui_and_logs( "Error occurred while decoding output.json" ) - except node_ports.exceptions.NodeportsException: + except node_ports_v2.exceptions.NodeportsException: await self._error_message_to_ui_and_logs( "Error occurred while setting port" ) diff --git a/services/sidecar/src/simcore_service_sidecar/rabbitmq.py b/services/sidecar/src/simcore_service_sidecar/rabbitmq.py index e460e2a16efa..1576c0b14b64 100644 --- a/services/sidecar/src/simcore_service_sidecar/rabbitmq.py +++ b/services/sidecar/src/simcore_service_sidecar/rabbitmq.py @@ -90,6 +90,7 @@ async def close(self): async def _post_message( self, exchange: aio_pika.Exchange, data: Dict[str, Union[str, Any]] ): + log.debug("publishing message to the broker %s", data) await exchange.publish( aio_pika.Message(body=json.dumps(data).encode()), routing_key="" ) diff --git a/services/sidecar/tests/integration/test_sidecar.py b/services/sidecar/tests/integration/test_sidecar.py index e1c67a236df9..079fe05d75d6 100644 --- a/services/sidecar/tests/integration/test_sidecar.py +++ b/services/sidecar/tests/integration/test_sidecar.py @@ -193,7 +193,7 @@ async def pipeline( """creates a full pipeline. NOTE: 'pipeline', defined as parametrization """ - from simcore_sdk import node_ports + from simcore_sdk import node_ports_v2 tasks = {key: osparc_service for key in pipeline_cfg} dag = {key: pipeline_cfg[key]["next"] for key in pipeline_cfg} @@ -233,15 +233,15 @@ async def _create( ): # update the files in mock_dir to S3 # FIXME: node_ports config shall not global! here making a hack so it works - node_ports.node_config.USER_ID = user_id - node_ports.node_config.PROJECT_ID = project_id - node_ports.node_config.NODE_UUID = node_uuid + node_ports_v2.node_config.USER_ID = user_id + node_ports_v2.node_config.PROJECT_ID = project_id + node_ports_v2.node_config.NODE_UUID = node_uuid print("--" * 10) - print_module_variables(module=node_ports.node_config) + print_module_variables(module=node_ports_v2.node_config) print("--" * 10) - PORTS = await node_ports.ports() + PORTS = await node_ports_v2.ports() await (await PORTS.inputs)[input_key].set( mock_dir / node_inputs[input_key]["path"] ) From e1a3703e27bbf908622d39fdd9b30afa7c5d5209 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 14:25:26 +0100 Subject: [PATCH 56/74] add tqdm to time download --- packages/simcore-sdk/requirements/_base.in | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in index 6cc3d313f020..c07e8eb206fd 100644 --- a/packages/simcore-sdk/requirements/_base.in +++ b/packages/simcore-sdk/requirements/_base.in @@ -14,6 +14,7 @@ networkx psycopg2-binary pydantic[email] tenacity +tqdm trafaret-config attrs From 3825467fe5b0a35fde03449c4d339b406fd6cd02 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 15:03:25 +0100 Subject: [PATCH 57/74] time download upload using tqdm --- .../src/simcore_sdk/node_ports/filemanager.py | 60 ++++++++++++------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py index b93508d0808d..aed3030ec1d4 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py @@ -12,6 +12,7 @@ from models_library.settings.services_common import ServicesCommonSettings from simcore_service_storage_sdk import ApiClient, Configuration, UsersApi from simcore_service_storage_sdk.rest import ApiException +from tqdm import tqdm from ..config.http_clients import client_request_settings from . import config, exceptions @@ -150,13 +151,22 @@ async def _download_link_to_file(session: ClientSession, url: URL, file_path: Pa if response.status > 299: raise exceptions.TransferError(url) file_path.parent.mkdir(parents=True, exist_ok=True) - async with aiofiles.open(file_path, "wb") as file_pointer: - # await file_pointer.write(await response.read()) - chunk = await response.content.read(CHUNK_SIZE) - while chunk: - await file_pointer.write(chunk) + file_size = int(response.headers.get("content-length", 0)) or None + + with tqdm( + desc=f"downloading {file_path} [{file_size} bytes]", + total=file_size, + unit="byte", + unit_scale=True, + ) as pbar: + async with aiofiles.open(file_path, "wb") as file_pointer: + # await file_pointer.write(await response.read()) chunk = await response.content.read(CHUNK_SIZE) - log.debug("Download complete") + while chunk: + await file_pointer.write(chunk) + pbar.update(len(chunk)) + chunk = await response.content.read(CHUNK_SIZE) + log.debug("Download complete") return await response.release() @@ -167,21 +177,29 @@ async def _upload_file_to_link( session: ClientSession, url: URL, file_path: Path ) -> Optional[ETag]: log.debug("Uploading from %s to %s", file_path, url) - async with session.put(url, data=file_path.open("rb")) as resp: - if resp.status > 299: - response_text = await resp.text() - raise exceptions.S3TransferError( - "Could not upload file {}:{}".format(file_path, response_text) - ) - if resp.status != 200: - response_text = await resp.text() - raise exceptions.S3TransferError( - "Issue when uploading file {}:{}".format(file_path, response_text) - ) - # get the S3 etag from the headers - e_tag = resp.headers.get("Etag", None) - log.debug("Uploaded %s to %s, received Etag %s", file_path, url, e_tag) - return e_tag + file_size = file_path.stat().st_size + with tqdm( + desc=f"uploading {file_path} [{file_size} bytes]", + total=file_size, + unit="byte", + unit_scale=True, + ) as pbar: + async with session.put(url, data=file_path.open("rb")) as resp: + if resp.status > 299: + response_text = await resp.text() + raise exceptions.S3TransferError( + "Could not upload file {}:{}".format(file_path, response_text) + ) + if resp.status != 200: + response_text = await resp.text() + raise exceptions.S3TransferError( + "Issue when uploading file {}:{}".format(file_path, response_text) + ) + pbar.update(file_size) + # get the S3 etag from the headers + e_tag = resp.headers.get("Etag", None) + log.debug("Uploaded %s to %s, received Etag %s", file_path, url, e_tag) + return e_tag async def download_file_from_s3( From c0124df9dd605a869fd559c68a415974d1715151 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 15:25:07 +0100 Subject: [PATCH 58/74] linter --- services/sidecar/src/simcore_service_sidecar/executor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/sidecar/src/simcore_service_sidecar/executor.py b/services/sidecar/src/simcore_service_sidecar/executor.py index ccc6ceaa8aca..a74d3354592f 100644 --- a/services/sidecar/src/simcore_service_sidecar/executor.py +++ b/services/sidecar/src/simcore_service_sidecar/executor.py @@ -309,6 +309,7 @@ async def _start_monitoring_container( ) return log_processor_task + # pylint: disable=too-many-statements async def _run_container(self): start_time = time.perf_counter() docker_image = f"{config.DOCKER_REGISTRY}/{self.task.image['name']}:{self.task.image['tag']}" From 208d1a5be67da439f5213209cd0fe0308504358e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 23:20:29 +0100 Subject: [PATCH 59/74] add bumpversion needed files --- packages/simcore-sdk/setup.cfg | 19 +++++++++++++++++++ .../simcore-sdk/src/simcore_sdk/__init__.py | 5 +++++ 2 files changed, 24 insertions(+) create mode 100644 packages/simcore-sdk/setup.cfg diff --git a/packages/simcore-sdk/setup.cfg b/packages/simcore-sdk/setup.cfg new file mode 100644 index 000000000000..43d3dea15635 --- /dev/null +++ b/packages/simcore-sdk/setup.cfg @@ -0,0 +1,19 @@ +[bumpversion] +current_version = 0.2.0 +commit = True +tag = False + +[bumpversion:file:setup.py] +search = version='{current_version}' +replace = version='{new_version}' + +[bumpversion:file:src/simcore_sdk/__init__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bdist_wheel] +universal = 1 + +[aliases] +# Define setup.py command aliases here +test = pytest diff --git a/packages/simcore-sdk/src/simcore_sdk/__init__.py b/packages/simcore-sdk/src/simcore_sdk/__init__.py index e69de29bb2d1..d6b5e03630a9 100644 --- a/packages/simcore-sdk/src/simcore_sdk/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/__init__.py @@ -0,0 +1,5 @@ +""" osparc's simcore-sdk library + +""" + +__version__ = "0.2.0" From c1cce6a2e541b35c73ac4698f355a08901679c01 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 8 Dec 2020 23:25:27 +0100 Subject: [PATCH 60/74] bumped version to 0.3.0 --- packages/simcore-sdk/setup.cfg | 2 +- packages/simcore-sdk/setup.py | 2 +- packages/simcore-sdk/src/simcore_sdk/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/simcore-sdk/setup.cfg b/packages/simcore-sdk/setup.cfg index 43d3dea15635..ca374b03ae10 100644 --- a/packages/simcore-sdk/setup.cfg +++ b/packages/simcore-sdk/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.0 +current_version = 0.3.0 commit = True tag = False diff --git a/packages/simcore-sdk/setup.py b/packages/simcore-sdk/setup.py index a832795eb719..8214ad50315b 100644 --- a/packages/simcore-sdk/setup.py +++ b/packages/simcore-sdk/setup.py @@ -24,7 +24,7 @@ def read_reqs(reqs_path: Path): setup( name="simcore-sdk", - version="0.2.0", + version="0.3.0", packages=find_packages(where="src"), package_dir={"": "src"}, python_requires=">=3.6", diff --git a/packages/simcore-sdk/src/simcore_sdk/__init__.py b/packages/simcore-sdk/src/simcore_sdk/__init__.py index d6b5e03630a9..11e5f72daded 100644 --- a/packages/simcore-sdk/src/simcore_sdk/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/__init__.py @@ -2,4 +2,4 @@ """ -__version__ = "0.2.0" +__version__ = "0.3.0" From 36473c9b1edc171a330021475ea5ecca3611dd9b Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 08:49:52 +0100 Subject: [PATCH 61/74] fix bad merge --- packages/simcore-sdk/requirements/_base.txt | 1 + packages/simcore-sdk/requirements/_test.txt | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 6f9183fee4ab..b954e46a8e30 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -40,6 +40,7 @@ six==1.15.0 # via isodate, jsonschema, openapi-core, openapi-spec- sqlalchemy[postgresql_psycopg2binary]==1.3.20 # via -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt, -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt, -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt, -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt, -c requirements/../../../requirements/constraints.txt, -r requirements/../../../packages/postgres-database/requirements/_base.in, -r requirements/../../../packages/service-library/requirements/_base.in, aiopg strict-rfc3339==0.7 # via openapi-core tenacity==6.2.0 # via -r requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in +tqdm==4.54.1 # via -r requirements/_base.in trafaret-config==2.0.2 # via -r requirements/_base.in trafaret==2.1.0 # via -r requirements/../../../packages/service-library/requirements/_base.in, trafaret-config typing-extensions==3.7.4.3 # via aiohttp, yarl diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index ed1163967993..eee9ec2777f1 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -25,6 +25,8 @@ importlib-metadata==3.1.1 # via -c requirements/_base.txt, pluggy, pytest iniconfig==1.1.1 # via pytest isort==5.6.4 # via pylint lazy-object-proxy==1.4.3 # via -c requirements/_base.txt, astroid +mako==1.1.3 # via alembic +markupsafe==1.1.1 # via mako mccabe==0.6.1 # via pylint multidict==5.1.0 # via -c requirements/_base.txt, aiohttp, yarl packaging==20.7 # via pytest, pytest-sugar @@ -44,7 +46,7 @@ pytest-runner==5.2 # via -r requirements/_test.in pytest-sugar==0.9.4 # via -r requirements/_test.in pytest-xdist==2.1.0 # via -r requirements/_test.in pytest==6.1.2 # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-forked, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar, pytest-xdist -python-dateutil==2.8.1 # via alembic +python-dateutil==2.8.1 # via -c requirements/_base.txt, alembic python-dotenv==0.15.0 # via -r requirements/_test.in python-editor==1.0.4 # via alembic requests==2.25.0 # via -r requirements/_test.in, coveralls, docker From 70814a284d79ca56446de8f581506c687aa93ae4 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 08:57:14 +0100 Subject: [PATCH 62/74] @pcrespov review: add deprecation warnings --- .../src/simcore_sdk/node_ports/__init__.py | 5 +++++ .../src/simcore_sdk/node_ports/nodeports.py | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py index 9a2fd0a3f2b4..9240286dda73 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py @@ -1,4 +1,5 @@ import logging +import warnings from . import config as node_config from . import exceptions @@ -9,3 +10,7 @@ # in that sense it should not log stuff unless the application code wants it to be so. log = logging.getLogger(__name__) log.addHandler(logging.NullHandler()) + +warnings.warn( + "node_ports is deprecated, use node_ports_v2 instead", category=DeprecationWarning +) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py index 1710923d1b06..223a95d94d63 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py @@ -3,6 +3,11 @@ """ import logging + +# pylint: disable=missing-docstring +# pylint: disable=too-many-instance-attributes +# pylint: disable=too-many-arguments +import warnings from pathlib import Path from typing import Optional @@ -13,10 +18,6 @@ log = logging.getLogger(__name__) -# pylint: disable=missing-docstring -# pylint: disable=too-many-instance-attributes -# pylint: disable=too-many-arguments - class Nodeports: """Allows the client to access the inputs and outputs assigned to the node""" @@ -28,7 +29,10 @@ def __init__( input_payloads: DataItemsList = None, outputs_payloads: DataItemsList = None, ): - + warnings.warn( + "node_ports is deprecated, use node_ports_v2 instead", + category=DeprecationWarning, + ) log.debug( "Initialising Nodeports object with inputs %s and outputs %s", input_payloads, @@ -163,6 +167,10 @@ async def _get_node_from_node_uuid(self, node_uuid: str): async def ports(db_manager: Optional[dbmanager.DBManager] = None) -> Nodeports: + warnings.warn( + "node_ports is deprecated, use node_ports_v2 instead", + category=DeprecationWarning, + ) # FIXME: warning every dbmanager create a new db engine! if db_manager is None: # NOTE: keeps backwards compatibility db_manager = dbmanager.DBManager() From a64afa43246212c6f9c53908c54fb8355d21152e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 08:58:13 +0100 Subject: [PATCH 63/74] @pcrespov review: remove unnecessary exclusion --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ebb4d2af5781..26605c9c3fee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks -exclude: "^.venv$|^.cache$|^.pytest_cache$|^.*/_item.py$" +exclude: "^.venv$|^.cache$|^.pytest_cache$" default_language_version: python: python3.6 repos: From c16668806e856c31d706dc33646bf5d075a553ef Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 09:00:33 +0100 Subject: [PATCH 64/74] @pcrespov review: rename mock fixtures from subsystem to service --- .../services_api_mocks_for_aiohttp_clients.py | 4 ++-- packages/simcore-sdk/tests/unit/test_port.py | 2 +- .../tests/integration/computation/test_rabbit.py | 2 +- .../tests/integration/test_garbage_collection.py | 6 +++--- .../tests/integration/test_project_workflow.py | 4 ++-- .../unit/with_dbs/fast/test_access_to_studies.py | 5 ++--- .../tests/unit/with_dbs/fast/test_director_v2.py | 11 +++++------ .../unit/with_dbs/medium/test_resource_manager.py | 6 +++--- .../server/tests/unit/with_dbs/slow/test_projects.py | 12 +++++++----- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py index e36b9195920b..1576528780c0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py +++ b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py @@ -25,7 +25,7 @@ def creation_cb(url, **kwargs) -> CallbackResult: @pytest.fixture -async def director_v2_subsystem_mock() -> aioresponses: +async def director_v2_service_mock() -> aioresponses: """uses aioresponses to mock all calls of an aiohttpclient WARNING: any request done through the client will go through aioresponses. It is @@ -67,7 +67,7 @@ async def director_v2_subsystem_mock() -> aioresponses: @pytest.fixture -async def storage_v0_subsystem_mock() -> aioresponses: +async def storage_v0_service_mock() -> aioresponses: """uses aioresponses to mock all calls of an aiohttpclient WARNING: any request done through the client will go through aioresponses. It is diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py index 9425c3f2cd44..b688e19775e9 100644 --- a/packages/simcore-sdk/tests/unit/test_port.py +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -168,7 +168,7 @@ async def mock_upload_file(mocker, e_tag): @pytest.fixture(autouse=True) def common_fixtures( loop, - storage_v0_subsystem_mock, + storage_v0_service_mock, mock_download_file, mock_upload_file, project_id: str, diff --git a/services/web/server/tests/integration/computation/test_rabbit.py b/services/web/server/tests/integration/computation/test_rabbit.py index 4e864d342c7b..cd2c44ea2f2c 100644 --- a/services/web/server/tests/integration/computation/test_rabbit.py +++ b/services/web/server/tests/integration/computation/test_rabbit.py @@ -188,7 +188,7 @@ async def _wait_until(pred: Callable, timeout: int): ], ) async def test_rabbit_websocket_computation( - director_v2_subsystem_mock, + director_v2_service_mock, mock_orphaned_services, logged_user, user_project, diff --git a/services/web/server/tests/integration/test_garbage_collection.py b/services/web/server/tests/integration/test_garbage_collection.py index a13409ba3dd6..86b02db3873c 100644 --- a/services/web/server/tests/integration/test_garbage_collection.py +++ b/services/web/server/tests/integration/test_garbage_collection.py @@ -76,7 +76,7 @@ async def __delete_all_redis_keys__(redis_service: RedisConfig): @pytest.fixture -async def director_v2_subsystem_mock() -> aioresponses: +async def director_v2_service_mock() -> aioresponses: """uses aioresponses to mock all calls of an aiohttpclient WARNING: any request done through the client will go through aioresponses. It is unfortunate but that means any valid request (like calling the test server) prefix must be set as passthrough. @@ -103,9 +103,9 @@ async def director_v2_subsystem_mock() -> aioresponses: @pytest.fixture(autouse=True) async def auto_mock_director_v2( - director_v2_subsystem_mock: aioresponses, + director_v2_service_mock: aioresponses, ) -> aioresponses: - return director_v2_subsystem_mock + return director_v2_service_mock @pytest.fixture diff --git a/services/web/server/tests/integration/test_project_workflow.py b/services/web/server/tests/integration/test_project_workflow.py index 994f6fabab85..d8c0d04ad0cd 100644 --- a/services/web/server/tests/integration/test_project_workflow.py +++ b/services/web/server/tests/integration/test_project_workflow.py @@ -250,7 +250,7 @@ async def test_workflow( primary_group: Dict[str, str], standard_groups: List[Dict[str, str]], storage_subsystem_mock, - director_v2_subsystem_mock, + director_v2_service_mock, ): # empty list projects = await _request_list(client) @@ -374,7 +374,7 @@ async def test_list_template_projects( fake_template_projects_isan, fake_template_projects_osparc, catalog_subsystem_mock, - director_v2_subsystem_mock, + director_v2_service_mock, ): catalog_subsystem_mock( fake_template_projects diff --git a/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py b/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py index 3552565d7782..36c63854e7bf 100644 --- a/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py +++ b/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py @@ -17,7 +17,6 @@ from aiohttp import ClientResponse, ClientSession, web from aiohttp.test_utils import TestClient from aioresponses import aioresponses - from models_library.projects_state import ( Owner, ProjectLocked, @@ -135,8 +134,8 @@ async def unpublished_project(client, fake_project): @pytest.fixture(autouse=True) -async def director_v2_mock(director_v2_subsystem_mock) -> aioresponses: - yield director_v2_subsystem_mock +async def director_v2_mock(director_v2_service_mock) -> aioresponses: + yield director_v2_service_mock async def _get_user_projects(client): diff --git a/services/web/server/tests/unit/with_dbs/fast/test_director_v2.py b/services/web/server/tests/unit/with_dbs/fast/test_director_v2.py index bb4fe0096c41..76482506db6d 100644 --- a/services/web/server/tests/unit/with_dbs/fast/test_director_v2.py +++ b/services/web/server/tests/unit/with_dbs/fast/test_director_v2.py @@ -5,14 +5,13 @@ from typing import Dict from uuid import UUID, uuid4 -from models_library.projects_state import RunningState -from pydantic.types import PositiveInt import pytest +from _helpers import ExpectedResponse, standard_role_response from aiohttp import web from aioresponses import aioresponses - -from _helpers import ExpectedResponse, standard_role_response +from models_library.projects_state import RunningState +from pydantic.types import PositiveInt from pytest_simcore.helpers.utils_assert import assert_status from pytest_simcore.helpers.utils_login import LoggedUser from simcore_service_webserver import director_v2 @@ -38,9 +37,9 @@ async def logged_user(client, user_role: UserRole): @pytest.fixture(autouse=True) async def auto_mock_director_v2( loop, - director_v2_subsystem_mock: aioresponses, + director_v2_service_mock: aioresponses, ) -> aioresponses: - yield director_v2_subsystem_mock + yield director_v2_service_mock @pytest.fixture diff --git a/services/web/server/tests/unit/with_dbs/medium/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/medium/test_resource_manager.py index 5d229dc64de9..c926be5d4195 100644 --- a/services/web/server/tests/unit/with_dbs/medium/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/medium/test_resource_manager.py @@ -107,8 +107,8 @@ async def empty_user_project2(client, empty_project, logged_user): @pytest.fixture(autouse=True) -async def director_v2_mock(director_v2_subsystem_mock) -> aioresponses: - yield director_v2_subsystem_mock +async def director_v2_mock(director_v2_service_mock) -> aioresponses: + yield director_v2_service_mock # ------------------------ UTILS ---------------------------------- @@ -333,7 +333,7 @@ async def test_interactive_services_removed_after_logout( client_session_id, socketio_client, storage_subsystem_mock, # when guest user logs out garbage is collected - director_v2_subsystem_mock: aioresponses, + director_v2_service_mock: aioresponses, ): set_service_deletion_delay(SERVICE_DELETION_DELAY, client.server.app) # login - logged_user fixture diff --git a/services/web/server/tests/unit/with_dbs/slow/test_projects.py b/services/web/server/tests/unit/with_dbs/slow/test_projects.py index b13f3563d466..dbd06d2f230c 100644 --- a/services/web/server/tests/unit/with_dbs/slow/test_projects.py +++ b/services/web/server/tests/unit/with_dbs/slow/test_projects.py @@ -13,9 +13,12 @@ import pytest import socketio -from _helpers import ExpectedResponse, HTTPLocked, standard_role_response from aiohttp import web from aioresponses import aioresponses +from socketio.exceptions import ConnectionError as SocketConnectionError + +from _helpers import ExpectedResponse, HTTPLocked, standard_role_response +from mock import call from models_library.projects_access import Owner from models_library.projects_state import ( ProjectLocked, @@ -47,7 +50,6 @@ from simcore_service_webserver.socketio.events import SOCKET_IO_PROJECT_UPDATED_EVENT from simcore_service_webserver.tags import setup_tags from simcore_service_webserver.utils import now_str, to_datetime -from socketio.exceptions import ConnectionError as SocketConnectionError API_VERSION = "v0" RESOURCE_NAME = "projects" @@ -233,9 +235,9 @@ async def mocked_get_services_for_user(*args, **kwargs): @pytest.fixture(autouse=True) async def director_v2_automock( - director_v2_subsystem_mock: aioresponses, + director_v2_service_mock: aioresponses, ) -> aioresponses: - yield director_v2_subsystem_mock + yield director_v2_service_mock # HELPERS ----------------------------------------------------------------------------------------- @@ -525,7 +527,7 @@ async def test_list_projects( template_project, expected, catalog_subsystem_mock, - director_v2_subsystem_mock, + director_v2_service_mock, ): catalog_subsystem_mock([user_project, template_project]) data = await _list_projects(client, expected) From 5136eda4af2a18aca558588ccd9cd77a6fb0d1ee Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 09:21:06 +0100 Subject: [PATCH 65/74] @pcrespov review: missing comments --- .../simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py index 02150bfd2ab8..0b72695d2af7 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -31,7 +31,7 @@ async def get_value_from_link( file_path = data_items_utils.create_file_path(key, file_name) if value == file_path: - # this can happen in case + # this is a corner case: in case the output key of the other node has the same name as the input key return value if file_path.exists(): file_path.unlink() From 80439777454294d6cd0e692d495d06baf675596e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 09:26:57 +0100 Subject: [PATCH 66/74] @pcrespov review: use dump/load names --- .../src/simcore_sdk/node_ports_v2/__init__.py | 6 ++--- .../node_ports_v2/serialization_v2.py | 8 +++---- .../tests/unit/test_serialization_v2.py | 23 ++++++++----------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py index cd96448bbd09..0c9ab793c8e6 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py @@ -6,7 +6,7 @@ from ..node_ports.dbmanager import DBManager from .nodeports_v2 import Nodeports from .port import Port -from .serialization_v2 import create_nodeports_from_db +from .serialization_v2 import load # nodeports is a library for accessing data linked to the node # in that sense it should not log stuff unless the application code wants it to be so. @@ -19,9 +19,7 @@ async def ports(db_manager: Optional[DBManager] = None) -> Nodeports: if db_manager is None: # NOTE: keeps backwards compatibility db_manager = DBManager() - return await create_nodeports_from_db( - db_manager, node_config.NODE_UUID, auto_update=True - ) + return await load(db_manager, node_config.NODE_UUID, auto_update=True) __all__ = ["ports", "node_config", "exceptions", "Port"] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py index 0580f369c27a..9c2dac47f5cb 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -14,7 +14,7 @@ } -async def create_nodeports_from_db( +async def load( db_manager: DBManager, node_uuid: str, auto_update: bool = False ) -> Nodeports: """creates a nodeport object from a row from comp_tasks""" @@ -46,14 +46,14 @@ async def create_nodeports_from_db( **node_ports_cfg, db_manager=db_manager, node_uuid=node_uuid, - save_to_db_cb=save_nodeports_to_db, - node_port_creator_cb=create_nodeports_from_db, + save_to_db_cb=dump, + node_port_creator_cb=load, auto_update=auto_update, ) return ports -async def save_nodeports_to_db(nodeports: Nodeports) -> None: +async def dump(nodeports: Nodeports) -> None: _nodeports_cfg = nodeports.dict( include={"internal_inputs", "internal_outputs"}, by_alias=True, diff --git a/packages/simcore-sdk/tests/unit/test_serialization_v2.py b/packages/simcore-sdk/tests/unit/test_serialization_v2.py index 9833f5146cd1..3460be302077 100644 --- a/packages/simcore-sdk/tests/unit/test_serialization_v2.py +++ b/packages/simcore-sdk/tests/unit/test_serialization_v2.py @@ -6,45 +6,42 @@ import pytest from simcore_sdk.node_ports_v2 import DBManager, exceptions -from simcore_sdk.node_ports_v2.serialization_v2 import ( - create_nodeports_from_db, - save_nodeports_to_db, -) +from simcore_sdk.node_ports_v2.serialization_v2 import dump, load @pytest.mark.parametrize("auto_update", [True, False]) -async def test_create_nodeports_from_db( +async def test_load( mock_db_manager, auto_update: bool, node_uuid: str, default_configuration: Dict[str, Any], ): db_manager: DBManager = mock_db_manager(default_configuration) - node_ports = await create_nodeports_from_db(db_manager, node_uuid, auto_update) + node_ports = await load(db_manager, node_uuid, auto_update) assert node_ports.db_manager == db_manager assert node_ports.node_uuid == node_uuid # pylint: disable=comparison-with-callable - assert node_ports.save_to_db_cb == save_nodeports_to_db - assert node_ports.node_port_creator_cb == create_nodeports_from_db + assert node_ports.save_to_db_cb == dump + assert node_ports.node_port_creator_cb == load assert node_ports.auto_update == auto_update -async def test_create_nodeports_from_db_with_invalid_cfg( +async def test_load_with_invalid_cfg( mock_db_manager, node_uuid: str, ): invalid_config = {"bad_key": "bad_value"} db_manager: DBManager = mock_db_manager(invalid_config) with pytest.raises(exceptions.InvalidProtocolError): - _ = await create_nodeports_from_db(db_manager, node_uuid) + _ = await load(db_manager, node_uuid) -async def test_save_nodeports_to_db( +async def test_dump( mock_db_manager, node_uuid: str, default_configuration: Dict[str, Any], ): db_manager: DBManager = mock_db_manager(default_configuration) - node_ports = await create_nodeports_from_db(db_manager, node_uuid) + node_ports = await load(db_manager, node_uuid) - await save_nodeports_to_db(node_ports) + await dump(node_ports) From 031c3227cf18257df23ad31bc114ebb2d58bf9c7 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 09:27:48 +0100 Subject: [PATCH 67/74] @pcrespov review: typo --- packages/simcore-sdk/tests/unit/test_nodeports_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/simcore-sdk/tests/unit/test_nodeports_v2.py b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py index 735f8bfc11d1..841b4b22ce7e 100644 --- a/packages/simcore-sdk/tests/unit/test_nodeports_v2.py +++ b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py @@ -19,7 +19,7 @@ pytest.param(False, id="Autoupdate disabled"), ], ) -async def test_nodeports_aut_updates( +async def test_nodeports_auto_updates( mock_db_manager: Callable, default_configuration: Dict[str, Any], node_uuid: str, From 7aa4361f703adc78b7b9bd5651be056191a41279 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 10:11:30 +0100 Subject: [PATCH 68/74] @GitHK review: remove useless return --- .../simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 4b77549ae00d..a4da941a195b 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -64,7 +64,7 @@ async def set(self, item_key: str, item_value: ItemConcreteValue) -> None: # not available try outputs pass # if this fails it will raise an exception - return await (await self.outputs)[item_key].set(item_value) + await (await self.outputs)[item_key].set(item_value) async def set_file_by_keymap(self, item_value: Path) -> None: for output in (await self.outputs).values(): From ec4601ac4aa1956c32ceedfcfb3a7fafd9c3886f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 10:48:44 +0100 Subject: [PATCH 69/74] @GitHK review: check if while reading something bad happens --- .../src/simcore_sdk/node_ports/filemanager.py | 54 ++++++------------- 1 file changed, 17 insertions(+), 37 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py index aed3030ec1d4..40debd6ac17b 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py @@ -3,10 +3,10 @@ import warnings from contextlib import contextmanager from pathlib import Path -from typing import Any, Dict, Optional, Tuple +from typing import Optional, Tuple import aiofiles -from aiohttp import ClientSession, ClientTimeout +from aiohttp import ClientPayloadError, ClientSession, ClientTimeout from yarl import URL from models_library.settings.services_common import ServicesCommonSettings @@ -88,8 +88,6 @@ async def _get_location_id_from_location_name(store: str, api: UsersApi): raise exceptions.S3InvalidStore(store) except ApiException as err: _handle_api_exception(store, err) - if resp.error: - raise exceptions.StorageConnectionError(store, resp.error.to_str()) async def _get_link(store_id: int, file_id: str, apifct) -> URL: @@ -125,24 +123,6 @@ async def _get_upload_link(store_id: int, file_id: str, api: UsersApi) -> URL: return await _get_link(store_id, file_id, api.upload_file) -async def _get_file_metadata(store_id: int, file_id: str) -> Dict[str, Any]: - log.debug("Getting file metadata from store id [%s] for %s", store_id, file_id) - try: - with api_client() as client: - api = UsersApi(client) - resp = await api.get_file_metadata( - file_id=file_id, location_id=store_id, user_id=config.USER_ID - ) - if resp.error: - raise exceptions.StorageServerIssue( - f"getting file {file_id} metadata failed: [{resp.error.to_str()}]" - ) - return resp.data - - except ApiException as err: - _handle_api_exception(store_id, err) - - async def _download_link_to_file(session: ClientSession, url: URL, file_path: Path): log.debug("Downloading from %s to %s", url, file_path) async with session.get(url) as response: @@ -152,22 +132,22 @@ async def _download_link_to_file(session: ClientSession, url: URL, file_path: Pa raise exceptions.TransferError(url) file_path.parent.mkdir(parents=True, exist_ok=True) file_size = int(response.headers.get("content-length", 0)) or None - - with tqdm( - desc=f"downloading {file_path} [{file_size} bytes]", - total=file_size, - unit="byte", - unit_scale=True, - ) as pbar: - async with aiofiles.open(file_path, "wb") as file_pointer: - # await file_pointer.write(await response.read()) - chunk = await response.content.read(CHUNK_SIZE) - while chunk: - await file_pointer.write(chunk) - pbar.update(len(chunk)) + try: + with tqdm( + desc=f"downloading {file_path} [{file_size} bytes]", + total=file_size, + unit="byte", + unit_scale=True, + ) as pbar: + async with aiofiles.open(file_path, "wb") as file_pointer: chunk = await response.content.read(CHUNK_SIZE) - log.debug("Download complete") - return await response.release() + while chunk: + await file_pointer.write(chunk) + pbar.update(len(chunk)) + chunk = await response.content.read(CHUNK_SIZE) + log.debug("Download complete") + except ClientPayloadError as exc: + raise exceptions.TransferError(url) from exc ETag = str From 0dbe66f9d6d08d5c8a2b3a2d1e3bfb2360b56a46 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 10:52:11 +0100 Subject: [PATCH 70/74] @GitHK review: re-enabling assertion --- packages/simcore-sdk/tests/integration/test_nodeports2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/simcore-sdk/tests/integration/test_nodeports2.py b/packages/simcore-sdk/tests/integration/test_nodeports2.py index b412ba0c3adf..a6b1662b2c26 100644 --- a/packages/simcore-sdk/tests/integration/test_nodeports2.py +++ b/packages/simcore-sdk/tests/integration/test_nodeports2.py @@ -183,8 +183,8 @@ async def test_port_file_accessors( PORTS = await node_ports_v2.ports() await check_config_valid(PORTS, config_dict) assert await (await PORTS.outputs)["out_34"].get() is None # check emptyness - # with pytest.raises(exceptions.S3InvalidPathError): - # await PORTS.inputs["in_1"].get() + with pytest.raises(exceptions.InvalidDownloadLinkError): + await (await PORTS.inputs)["in_1"].get() # this triggers an upload to S3 + configuration change await (await PORTS.outputs)["out_34"].set(item_value) From 7295f336ca290bb1310dfc56ccde656db9560561 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 11:16:06 +0100 Subject: [PATCH 71/74] @GitHK review: removed unnecessary nullhandler for library --- .../simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py index 0c9ab793c8e6..a64f9640b1aa 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py @@ -8,15 +8,14 @@ from .port import Port from .serialization_v2 import load -# nodeports is a library for accessing data linked to the node -# in that sense it should not log stuff unless the application code wants it to be so. log = logging.getLogger(__name__) -log.addHandler(logging.NullHandler()) async def ports(db_manager: Optional[DBManager] = None) -> Nodeports: + log.debug("creating node_ports_v2 object using provided dbmanager: %s", db_manager) # FIXME: warning every dbmanager create a new db engine! if db_manager is None: # NOTE: keeps backwards compatibility + log.debug("no db manager provided, creating one...") db_manager = DBManager() return await load(db_manager, node_config.NODE_UUID, auto_update=True) From d8fe51ce25f2906d321d379a15d335433f7ec17e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 11:16:17 +0100 Subject: [PATCH 72/74] added a bit of debug logs --- .../node_ports_v2/serialization_v2.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py index 9c2dac47f5cb..2e2d05c4a96a 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -1,4 +1,6 @@ import json +import logging +from pprint import pformat from typing import Any, Dict, Set from aiopg.sa.result import RowProxy @@ -7,6 +9,8 @@ from ..node_ports.exceptions import InvalidProtocolError from .nodeports_v2 import Nodeports +log = logging.getLogger(__name__) + NODE_REQUIRED_KEYS: Set[str] = { "schema", "inputs", @@ -18,6 +22,11 @@ async def load( db_manager: DBManager, node_uuid: str, auto_update: bool = False ) -> Nodeports: """creates a nodeport object from a row from comp_tasks""" + log.debug( + "creating node_ports_v2 object from node %s with auto_uptate %s", + node_uuid, + auto_update, + ) row: RowProxy = await db_manager.get_ports_configuration_from_node_uuid(node_uuid) port_cfg = json.loads(row) if any(k not in port_cfg for k in NODE_REQUIRED_KEYS): @@ -50,10 +59,18 @@ async def load( node_port_creator_cb=load, auto_update=auto_update, ) + log.debug( + "created node_ports_v2 object %s", + pformat(ports, indent=2), + ) return ports async def dump(nodeports: Nodeports) -> None: + log.debug( + "dumping node_ports_v2 object %s", + pformat(nodeports, indent=2), + ) _nodeports_cfg = nodeports.dict( include={"internal_inputs", "internal_outputs"}, by_alias=True, From 898c9c5cbb524f11ec72c9246c33175cfd504fce Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Wed, 9 Dec 2020 12:05:54 +0100 Subject: [PATCH 73/74] fixed sidecar test by using both versions of node ports --- .../sidecar/tests/integration/test_sidecar.py | 97 +++++++++++++------ 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/services/sidecar/tests/integration/test_sidecar.py b/services/sidecar/tests/integration/test_sidecar.py index 079fe05d75d6..d6c1107134d8 100644 --- a/services/sidecar/tests/integration/test_sidecar.py +++ b/services/sidecar/tests/integration/test_sidecar.py @@ -2,6 +2,7 @@ # pylint: disable=redefined-outer-name # pylint: disable=too-many-arguments import asyncio +import importlib import inspect import json from collections import deque @@ -178,7 +179,12 @@ async def _assert_incoming_data_logs( return (sidecar_logs, tasks_logs, progress_logs) -@pytest.fixture +@pytest.fixture( + params=[ + "node_ports", + "node_ports_v2", + ] +) async def pipeline( sidecar_config: None, postgres_host_config: Dict[str, str], @@ -189,16 +195,18 @@ async def pipeline( pipeline_cfg: Dict, mock_dir: Path, user_id: int, + request, ) -> ComputationalPipeline: """creates a full pipeline. NOTE: 'pipeline', defined as parametrization """ - from simcore_sdk import node_ports_v2 tasks = {key: osparc_service for key in pipeline_cfg} dag = {key: pipeline_cfg[key]["next"] for key in pipeline_cfg} inputs = {key: pipeline_cfg[key]["inputs"] for key in pipeline_cfg} + np = importlib.import_module(f".{request.param}", package="simcore_sdk") + async def _create( tasks: Dict[str, Any], dag: Dict[str, List[str]], @@ -233,15 +241,15 @@ async def _create( ): # update the files in mock_dir to S3 # FIXME: node_ports config shall not global! here making a hack so it works - node_ports_v2.node_config.USER_ID = user_id - node_ports_v2.node_config.PROJECT_ID = project_id - node_ports_v2.node_config.NODE_UUID = node_uuid + np.node_config.USER_ID = user_id + np.node_config.PROJECT_ID = project_id + np.node_config.NODE_UUID = node_uuid print("--" * 10) - print_module_variables(module=node_ports_v2.node_config) + print_module_variables(module=np.node_config) print("--" * 10) - PORTS = await node_ports_v2.ports() + PORTS = await np.ports() await (await PORTS.inputs)[input_key].set( mock_dir / node_inputs[input_key]["path"] ) @@ -254,29 +262,50 @@ async def _create( "itisfoundation/sleeper", "1.0.0", { - "node_1": { - "next": ["node_2", "node_3"], + "a13d197a-bf8c-4e11-8a15-44a9894cbbe8": { + "next": [ + "28bf052a-5fb8-4935-9c97-2b15109632b9", + "dfdc165b-a10d-4049-bf4e-555bf5e7d557", + ], "inputs": {}, }, - "node_2": { - "next": ["node_4"], + "28bf052a-5fb8-4935-9c97-2b15109632b9": { + "next": ["54901e30-6cd2-417b-aaf9-b458022639d2"], "inputs": { - "in_1": {"nodeUuid": "node_1", "output": "out_1"}, - "in_2": {"nodeUuid": "node_1", "output": "out_2"}, + "in_1": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "out_1", + }, + "in_2": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "out_2", + }, }, }, - "node_3": { - "next": ["node_4"], + "dfdc165b-a10d-4049-bf4e-555bf5e7d557": { + "next": ["54901e30-6cd2-417b-aaf9-b458022639d2"], "inputs": { - "in_1": {"nodeUuid": "node_1", "output": "out_1"}, - "in_2": {"nodeUuid": "node_1", "output": "out_2"}, + "in_1": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "out_1", + }, + "in_2": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "out_2", + }, }, }, - "node_4": { + "54901e30-6cd2-417b-aaf9-b458022639d2": { "next": [], "inputs": { - "in_1": {"nodeUuid": "node_2", "output": "out_1"}, - "in_2": {"nodeUuid": "node_3", "output": "out_2"}, + "in_1": { + "nodeUuid": "28bf052a-5fb8-4935-9c97-2b15109632b9", + "output": "out_1", + }, + "in_2": { + "nodeUuid": "dfdc165b-a10d-4049-bf4e-555bf5e7d557", + "output": "out_2", + }, }, }, }, @@ -286,25 +315,28 @@ async def _create( "itisfoundation/osparc-python-runner", "1.0.0", { - "node_1": { - "next": ["node_2", "node_3"], + "a13d197a-bf8c-4e11-8a15-44a9894cbbe8": { + "next": [ + "28bf052a-5fb8-4935-9c97-2b15109632b9", + "dfdc165b-a10d-4049-bf4e-555bf5e7d557", + ], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_sample.py"} }, }, - "node_2": { - "next": ["node_4"], + "28bf052a-5fb8-4935-9c97-2b15109632b9": { + "next": ["54901e30-6cd2-417b-aaf9-b458022639d2"], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_sample.py"} }, }, - "node_3": { - "next": ["node_4"], + "dfdc165b-a10d-4049-bf4e-555bf5e7d557": { + "next": ["54901e30-6cd2-417b-aaf9-b458022639d2"], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_sample.py"} }, }, - "node_4": { + "54901e30-6cd2-417b-aaf9-b458022639d2": { "next": [], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_sample.py"} @@ -319,18 +351,21 @@ async def _create( "itisfoundation/osparc-python-runner", "1.0.0", { - "node_1": { + "a13d197a-bf8c-4e11-8a15-44a9894cbbe8": { "next": [ - "node_2", + "28bf052a-5fb8-4935-9c97-2b15109632b9", ], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_factory.py"} }, }, - "node_2": { + "28bf052a-5fb8-4935-9c97-2b15109632b9": { "next": [], "inputs": { - "input_1": {"nodeUuid": "node_1", "output": "output_1"}, + "input_1": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "output_1", + }, }, }, }, From 904ca55ce9033dfc900ff146ba6b2365209c890a Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 14 Dec 2020 15:19:04 +0100 Subject: [PATCH 74/74] bad merge --- .../web/server/tests/unit/with_dbs/slow/test_projects.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/slow/test_projects.py b/services/web/server/tests/unit/with_dbs/slow/test_projects.py index dbd06d2f230c..19968fb77bcf 100644 --- a/services/web/server/tests/unit/with_dbs/slow/test_projects.py +++ b/services/web/server/tests/unit/with_dbs/slow/test_projects.py @@ -13,12 +13,9 @@ import pytest import socketio +from _helpers import ExpectedResponse, HTTPLocked, standard_role_response from aiohttp import web from aioresponses import aioresponses -from socketio.exceptions import ConnectionError as SocketConnectionError - -from _helpers import ExpectedResponse, HTTPLocked, standard_role_response -from mock import call from models_library.projects_access import Owner from models_library.projects_state import ( ProjectLocked, @@ -50,6 +47,7 @@ from simcore_service_webserver.socketio.events import SOCKET_IO_PROJECT_UPDATED_EVENT from simcore_service_webserver.tags import setup_tags from simcore_service_webserver.utils import now_str, to_datetime +from socketio.exceptions import ConnectionError as SocketConnectionError API_VERSION = "v0" RESOURCE_NAME = "projects"