diff --git a/.coveragerc b/.coveragerc index 786ba6c85c7e..21400449ce48 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,27 @@ [run] branch = True -omit = +omit = */tests/* */generated_code/* parallel = True + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +ignore_errors = True +show_missing = True diff --git a/api/specs/common/schemas/project-v0.0.1-converted.yaml b/api/specs/common/schemas/project-v0.0.1-converted.yaml index b4e0fbd30f4e..48c573471710 100644 --- a/api/specs/common/schemas/project-v0.0.1-converted.yaml +++ b/api/specs/common/schemas/project-v0.0.1-converted.yaml @@ -159,6 +159,8 @@ properties: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -221,6 +223,8 @@ properties: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: diff --git a/api/specs/common/schemas/project-v0.0.1.json b/api/specs/common/schemas/project-v0.0.1.json index d61d3cc2107c..a0ba4137bfdd 100644 --- a/api/specs/common/schemas/project-v0.0.1.json +++ b/api/specs/common/schemas/project-v0.0.1.json @@ -210,6 +210,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -302,6 +305,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -344,7 +350,10 @@ ] }, "parent": { - "type": [ "null", "string" ], + "type": [ + "null", + "string" + ], "format": "uuid", "description": "Parent's (group-nodes') node ID s.", "examples": [ diff --git a/packages/models-library/src/models_library/projects_nodes_io.py b/packages/models-library/src/models_library/projects_nodes_io.py index 305176b8ad3c..34bd937ea386 100644 --- a/packages/models-library/src/models_library/projects_nodes_io.py +++ b/packages/models-library/src/models_library/projects_nodes_io.py @@ -9,7 +9,6 @@ from .services import PROPERTY_KEY_RE - NodeID = UUID # Pydantic does not support exporting a jsonschema with Dict keys being something else than a str @@ -54,12 +53,18 @@ class BaseFileLink(BaseModel): ) path: str = Field( ..., + regex=r"^.+$", description="The path to the file in the storage provider domain", example=[ "N:package:b05739ef-260c-4038-b47d-0240d04b0599", "94453a6a-c8d4-52b3-a22d-ccbf81f8d636/d4442ca4-23fd-5b6b-ba6d-0b75f711c109/y_1D.txt", ], ) + e_tag: Optional[str] = Field( + None, + description="Entity tag that uniquely represents the file. The method to generate the tag is not specified (black box).", + alias="eTag", + ) class Config: extra = Extra.forbid diff --git a/packages/models-library/src/models_library/services.py b/packages/models-library/src/models_library/services.py index 99dbfc7718d7..73c89213dd65 100644 --- a/packages/models-library/src/models_library/services.py +++ b/packages/models-library/src/models_library/services.py @@ -6,7 +6,18 @@ from enum import Enum from typing import Dict, List, Optional, Union -from pydantic import BaseModel, EmailStr, Extra, Field, HttpUrl, constr, validator +from pydantic import ( + BaseModel, + EmailStr, + Extra, + Field, + HttpUrl, + StrictBool, + StrictFloat, + StrictInt, + constr, + validator, +) from pydantic.types import PositiveInt from .basic_regex import VERSION_RE @@ -150,7 +161,7 @@ class ServiceProperty(BaseModel): description="Place the data associated with the named keys in files", examples=[{"dir/input1.txt": "key_1", "dir33/input2.txt": "key2"}], ) - default_value: Optional[Union[str, float, bool, int]] = Field( + default_value: Optional[Union[StrictBool, StrictInt, StrictFloat, str]] = Field( None, alias="defaultValue", examples=["Dog", True] ) diff --git a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py index a81de63f5736..1576528780c0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py +++ b/packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py @@ -4,9 +4,10 @@ from aioresponses import aioresponses from aioresponses.core import CallbackResult from models_library.projects_state import RunningState +from yarl import URL -def creation_cb(url, **kwargs): +def creation_cb(url, **kwargs) -> CallbackResult: assert "json" in kwargs, f"missing body in call to {url}" body = kwargs["json"] @@ -24,7 +25,7 @@ def creation_cb(url, **kwargs): @pytest.fixture -async def director_v2_subsystem_mock() -> aioresponses: +async def director_v2_service_mock() -> aioresponses: """uses aioresponses to mock all calls of an aiohttpclient WARNING: any request done through the client will go through aioresponses. It is @@ -63,3 +64,38 @@ async def director_v2_subsystem_mock() -> aioresponses: mock.delete(delete_computation_pattern, status=204, repeat=True) yield mock + + +@pytest.fixture +async def storage_v0_service_mock() -> aioresponses: + + """uses aioresponses to mock all calls of an aiohttpclient + WARNING: any request done through the client will go through aioresponses. It is + unfortunate but that means any valid request (like calling the test server) prefix must be set as passthrough. + Other than that it seems to behave nicely + """ + PASSTHROUGH_REQUESTS_PREFIXES = ["http://127.0.0.1", "ws://"] + + def get_download_link_cb(url: URL, **kwargs) -> CallbackResult: + file_id = url.path.rsplit("/files/")[1] + + return CallbackResult( + status=200, payload={"data": {"link": f"file://{file_id}"}} + ) + + get_download_link_pattern = re.compile( + r"^http://[a-z\-_]*storage:[0-9]+/v0/locations/[0-9]+/files/.+$" + ) + + get_locations_link_pattern = re.compile( + r"^http://[a-z\-_]*storage:[0-9]+/v0/locations.*$" + ) + + with aioresponses(passthrough=PASSTHROUGH_REQUESTS_PREFIXES) as mock: + mock.get(get_download_link_pattern, callback=get_download_link_cb, repeat=True) + mock.get( + get_locations_link_pattern, + status=200, + payload={"data": [{"name": "simcore.s3", "id": "0"}]}, + ) + yield mock diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in index cd560ad2a6bb..c07e8eb206fd 100644 --- a/packages/simcore-sdk/requirements/_base.in +++ b/packages/simcore-sdk/requirements/_base.in @@ -12,8 +12,9 @@ aiohttp aiopg[sa] networkx psycopg2-binary -pydantic +pydantic[email] tenacity +tqdm trafaret-config attrs diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 6f9183fee4ab..b954e46a8e30 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -40,6 +40,7 @@ six==1.15.0 # via isodate, jsonschema, openapi-core, openapi-spec- sqlalchemy[postgresql_psycopg2binary]==1.3.20 # via -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt, -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt, -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt, -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt, -c requirements/../../../requirements/constraints.txt, -r requirements/../../../packages/postgres-database/requirements/_base.in, -r requirements/../../../packages/service-library/requirements/_base.in, aiopg strict-rfc3339==0.7 # via openapi-core tenacity==6.2.0 # via -r requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in +tqdm==4.54.1 # via -r requirements/_base.in trafaret-config==2.0.2 # via -r requirements/_base.in trafaret==2.1.0 # via -r requirements/../../../packages/service-library/requirements/_base.in, trafaret-config typing-extensions==3.7.4.3 # via aiohttp, yarl diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index 2abd0641c010..3f06bfbcc258 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -20,6 +20,7 @@ pytest-sugar pytest-xdist # mockups/fixtures +alembic aioresponses requests docker diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index 09bbd727d2a7..eee9ec2777f1 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -6,6 +6,7 @@ # aiohttp==3.7.3 # via -c requirements/_base.txt, aioresponses, pytest-aiohttp aioresponses==0.7.1 # via -r requirements/_test.in +alembic==1.4.3 # via -r requirements/_test.in apipkg==1.5 # via execnet astroid==2.4.2 # via pylint async-timeout==3.0.1 # via -c requirements/_base.txt, aiohttp @@ -24,11 +25,14 @@ importlib-metadata==3.1.1 # via -c requirements/_base.txt, pluggy, pytest iniconfig==1.1.1 # via pytest isort==5.6.4 # via pylint lazy-object-proxy==1.4.3 # via -c requirements/_base.txt, astroid +mako==1.1.3 # via alembic +markupsafe==1.1.1 # via mako mccabe==0.6.1 # via pylint multidict==5.1.0 # via -c requirements/_base.txt, aiohttp, yarl packaging==20.7 # via pytest, pytest-sugar pluggy==0.13.1 # via pytest pprintpp==0.4.0 # via pytest-icdiff +psycopg2-binary==2.8.6 # via -c requirements/_base.txt, sqlalchemy py==1.9.0 # via pytest, pytest-forked pylint==2.6.0 # via -r requirements/_test.in pyparsing==2.4.7 # via packaging @@ -42,9 +46,12 @@ pytest-runner==5.2 # via -r requirements/_test.in pytest-sugar==0.9.4 # via -r requirements/_test.in pytest-xdist==2.1.0 # via -r requirements/_test.in pytest==6.1.2 # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-forked, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar, pytest-xdist +python-dateutil==2.8.1 # via -c requirements/_base.txt, alembic python-dotenv==0.15.0 # via -r requirements/_test.in +python-editor==1.0.4 # via alembic requests==2.25.0 # via -r requirements/_test.in, coveralls, docker -six==1.15.0 # via -c requirements/_base.txt, astroid, docker, websocket-client +six==1.15.0 # via -c requirements/_base.txt, astroid, docker, python-dateutil, websocket-client +sqlalchemy[postgresql_psycopg2binary]==1.3.20 # via -c requirements/_base.txt, alembic termcolor==1.1.0 # via pytest-sugar toml==0.10.2 # via pylint, pytest typed-ast==1.4.1 # via astroid diff --git a/packages/simcore-sdk/setup.cfg b/packages/simcore-sdk/setup.cfg new file mode 100644 index 000000000000..ca374b03ae10 --- /dev/null +++ b/packages/simcore-sdk/setup.cfg @@ -0,0 +1,19 @@ +[bumpversion] +current_version = 0.3.0 +commit = True +tag = False + +[bumpversion:file:setup.py] +search = version='{current_version}' +replace = version='{new_version}' + +[bumpversion:file:src/simcore_sdk/__init__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bdist_wheel] +universal = 1 + +[aliases] +# Define setup.py command aliases here +test = pytest diff --git a/packages/simcore-sdk/setup.py b/packages/simcore-sdk/setup.py index a832795eb719..8214ad50315b 100644 --- a/packages/simcore-sdk/setup.py +++ b/packages/simcore-sdk/setup.py @@ -24,7 +24,7 @@ def read_reqs(reqs_path: Path): setup( name="simcore-sdk", - version="0.2.0", + version="0.3.0", packages=find_packages(where="src"), package_dir={"": "src"}, python_requires=">=3.6", diff --git a/packages/simcore-sdk/src/simcore_sdk/__init__.py b/packages/simcore-sdk/src/simcore_sdk/__init__.py index e69de29bb2d1..11e5f72daded 100644 --- a/packages/simcore-sdk/src/simcore_sdk/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/__init__.py @@ -0,0 +1,5 @@ +""" osparc's simcore-sdk library + +""" + +__version__ = "0.3.0" diff --git a/packages/simcore-sdk/src/simcore_sdk/config/db.py b/packages/simcore-sdk/src/simcore_sdk/config/db.py index 26b6d2206cf1..a67603685b8a 100644 --- a/packages/simcore-sdk/src/simcore_sdk/config/db.py +++ b/packages/simcore-sdk/src/simcore_sdk/config/db.py @@ -5,21 +5,22 @@ import trafaret as T -CONFIG_SCHEMA = T.Dict({ - "database": T.String(), - "user": T.String(), - "password": T.String(), - T.Key("minsize", default=1 ,optional=True): T.ToInt(), - T.Key("maxsize", default=4, optional=True): T.ToInt(), - "host": T.Or( T.String, T.Null), - "port": T.Or( T.ToInt, T.Null), - "endpoint": T.Or( T.String, T.Null) -}) +CONFIG_SCHEMA = T.Dict( + { + "database": T.String(), + "user": T.String(), + "password": T.String(), + T.Key("minsize", default=1, optional=True): T.ToInt(), + T.Key("maxsize", default=4, optional=True): T.ToInt(), + "host": T.Or(T.String, T.Null), + "port": T.Or(T.ToInt, T.Null), + "endpoint": T.Or(T.String, T.Null), + } +) # TODO: deprecate! -class Config(): - +class Config: def __init__(self): # TODO: uniform config classes . see server.config file POSTGRES_URL = env.get("POSTGRES_ENDPOINT", "postgres:5432") @@ -31,8 +32,9 @@ def __init__(self): self._pwd = POSTGRES_PW self._url = POSTGRES_URL self._db = POSTGRES_DB - self._endpoint = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}'.format( - user=self._user, pw=self._pwd, url=self._url, db=self._db) + self._endpoint = "postgresql+psycopg2://{user}:{pw}@{url}/{db}".format( + user=self._user, pw=self._pwd, url=self._url, db=self._db + ) @property def endpoint(self): diff --git a/packages/simcore-sdk/src/simcore_sdk/models/__init__.py b/packages/simcore-sdk/src/simcore_sdk/models/__init__.py index a2dcfc5df6cd..06f3450664c1 100644 --- a/packages/simcore-sdk/src/simcore_sdk/models/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/models/__init__.py @@ -1,8 +1,7 @@ from . import pipeline_models +from .base import metadata + # Add here new models -from .base import metadata -__all__ = ( - 'metadata' -) \ No newline at end of file +__all__ = "metadata" diff --git a/packages/simcore-sdk/src/simcore_sdk/models/base.py b/packages/simcore-sdk/src/simcore_sdk/models/base.py index 2431525cfa3f..72ba04fee4b4 100644 --- a/packages/simcore-sdk/src/simcore_sdk/models/base.py +++ b/packages/simcore-sdk/src/simcore_sdk/models/base.py @@ -1,5 +1,3 @@ from simcore_postgres_database.models.base import metadata -__all__ = [ - "metadata" -] +__all__ = ["metadata"] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py index 8fbf7dfbfe2a..595d6f1ed7c6 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_data/__init__.py @@ -1 +1 @@ -from . import data_manager \ No newline at end of file +from . import data_manager diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py index 8c9141e1dae4..9240286dda73 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/__init__.py @@ -1,4 +1,5 @@ import logging +import warnings from . import config as node_config from . import exceptions @@ -10,3 +11,6 @@ log = logging.getLogger(__name__) log.addHandler(logging.NullHandler()) +warnings.warn( + "node_ports is deprecated, use node_ports_v2 instead", category=DeprecationWarning +) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_item.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_item.py index 906a8bd8b1cb..6cd34cce2916 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_item.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_data_item.py @@ -2,23 +2,28 @@ import collections import logging +from typing import Dict, Optional, Union from . import config, exceptions log = logging.getLogger(__name__) +DataItemValue = Optional[Union[int, float, bool, str, Dict[str, Union[int, str]]]] + _DataItem = collections.namedtuple("_DataItem", config.DATA_ITEM_KEYS.keys()) + class DataItem(_DataItem): - """Encapsulates a Data Item and provides accessors functions + """Encapsulates a Data Item and provides accessors functions""" - """ def __new__(cls, **kwargs): new_kargs = dict.fromkeys(config.DATA_ITEM_KEYS.keys()) for key, required in config.DATA_ITEM_KEYS.items(): if key not in kwargs: if required: - raise exceptions.InvalidProtocolError(kwargs, "key \"%s\" is missing" % (str(key))) + raise exceptions.InvalidProtocolError( + kwargs, 'key "%s" is missing' % (str(key)) + ) new_kargs[key] = None else: new_kargs[key] = kwargs[key] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py index 6bb3aed15ebe..5b26338f6c7d 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_item.py @@ -6,13 +6,18 @@ from yarl import URL from . import config, data_items_utils, exceptions, filemanager -from ._data_item import DataItem +from ._data_item import DataItem, DataItemValue from ._schema_item import SchemaItem log = logging.getLogger(__name__) -def _check_type(item_type: str, value: Union[int, float, bool, str, Dict]): +ItemConcreteValue = Union[int, float, bool, str, Path] +NodeLink = Dict[str,str] +StoreLink = Dict[str,str] +DownloadLink = Dict[str,str] + +def _check_type(item_type: str, value: DataItemValue): if item_type not in config.TYPE_TO_PYTHON_TYPE_MAP and not data_items_utils.is_file_type(item_type): raise exceptions.InvalidItemTypeError(item_type, value) @@ -74,7 +79,7 @@ def __getattr__(self, name: str): raise AttributeError - async def get(self) -> Union[int, float, bool, str, Path]: + async def get(self) -> ItemConcreteValue: """ gets data converted to the underlying type :raises exceptions.InvalidProtocolError: if the underlying type is unknown @@ -124,7 +129,7 @@ async def get(self) -> Union[int, float, bool, str, Path]: config.TYPE_TO_PYTHON_TYPE_MAP[self.type]["converter"](self.value) ) - async def set(self, value: Union[int, float, bool, str, Path]): + async def set(self, value: ItemConcreteValue): """ sets the data to the underlying port :param value: must be convertible to a string, or an exception will be thrown. @@ -156,7 +161,7 @@ async def set(self, value: Union[int, float, bool, str, Path]): s3_object = data_items_utils.encode_file_id( file_path, project_id=config.PROJECT_ID, node_id=config.NODE_UUID ) - store_id = await filemanager.upload_file( + store_id, _ = await filemanager.upload_file( store_name=config.STORE, s3_object=s3_object, local_file_path=file_path ) log.debug("file path %s uploaded", value) @@ -171,7 +176,7 @@ async def set(self, value: Union[int, float, bool, str, Path]): log.debug("database updated") async def __get_value_from_link( - self, value: Dict[str, str] + self, value: NodeLink ) -> Union[int, float, bool, str, Path]: # pylint: disable=R1710 log.debug("Getting value %s", value) node_uuid, port_key = data_items_utils.decode_link(value) @@ -187,7 +192,7 @@ async def __get_value_from_link( log.debug("Received node from DB %s, now returning value", other_nodeports) return await other_nodeports.get(port_key) - async def __get_value_from_store(self, value: Dict[str, str]) -> Path: + async def __get_value_from_store(self, value: StoreLink) -> Path: log.debug("Getting value from storage %s", value) store_id, s3_path = data_items_utils.decode_store(value) # do not make any assumption about s3_path, it is a str containing stuff that can be anything depending on the store @@ -206,7 +211,7 @@ async def __get_value_from_store(self, value: Dict[str, str]) -> Path: return downloaded_file - async def _get_value_from_download_link(self, value: Dict[str,str]) -> Path: + async def _get_value_from_download_link(self, value: DownloadLink) -> Path: log.debug("Getting value from download link [%s] with label %s", value["downloadLink"], value.get("label", "undef")) download_link = URL(value["downloadLink"]) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_item.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_item.py index 87ed6a51a47d..b527169eda7a 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_item.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/_schema_item.py @@ -7,13 +7,16 @@ _SchemaItem = collections.namedtuple("_SchemaItem", config.SCHEMA_ITEM_KEYS.keys()) + class SchemaItem(_SchemaItem): def __new__(cls, **kwargs): new_kargs = dict.fromkeys(config.SCHEMA_ITEM_KEYS.keys()) for key, required in config.SCHEMA_ITEM_KEYS.items(): if key not in kwargs: if required: - raise exceptions.InvalidProtocolError(kwargs, "key \"%s\" is missing" % (str(key))) + raise exceptions.InvalidProtocolError( + kwargs, 'key "%s" is missing' % (str(key)) + ) # new_kargs[key] = None else: new_kargs[key] = kwargs[key] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py index f767bd2d77fa..cc0421410846 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/config.py @@ -22,7 +22,6 @@ # ------------------------------------------------------------------------- NODE_KEYS: Dict[str, bool] = { - "version": True, "schema": True, "inputs": True, "outputs": True, diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py index de9202e30fca..deec34502452 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/dbmanager.py @@ -5,8 +5,6 @@ import aiopg.sa import tenacity -from sqlalchemy import and_ - from servicelib.aiopg_utils import ( DataSourceName, PostgresRetryPolicyUponInitialization, @@ -14,9 +12,10 @@ is_postgres_responsive, ) from simcore_postgres_database.models.comp_tasks import comp_tasks -from .exceptions import NodeNotFound +from sqlalchemy import and_ from . import config +from .exceptions import NodeNotFound log = logging.getLogger(__name__) @@ -125,7 +124,6 @@ async def get_ports_configuration_from_node_uuid(self, node_uuid: str) -> str: node = await _get_node_from_db(node_uuid, connection) node_json_config = json.dumps( { - "version": "0.1", "schema": node.schema, "inputs": node.inputs, "outputs": node.outputs, diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py index f5e15714b012..318a135e1d8b 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/exceptions.py @@ -1,10 +1,10 @@ -"""Defines the different exceptions that may arise in the nodeports package""" +from typing import Optional class NodeportsException(Exception): """Basic exception for errors raised in nodeports""" - def __init__(self, msg=None): + def __init__(self, msg: Optional[str] = None): super().__init__(msg or "An error occured in simcore") @@ -16,21 +16,10 @@ def __init__(self, obj): self.obj = obj -class WrongProtocolVersionError(NodeportsException): - """Using wrong protocol version""" - - def __init__(self, expected_version, found_version): - super().__init__( - f"Expecting version {expected_version}, found version {found_version}" - ) - self.expected_version = expected_version - self.found_version = found_version - - class UnboundPortError(NodeportsException, IndexError): """Accessed port is not configured""" - def __init__(self, port_index, msg=None): + def __init__(self, port_index, msg: Optional[str] = None): super().__init__(f"No port bound at index {port_index}") self.port_index = port_index @@ -38,7 +27,7 @@ def __init__(self, port_index, msg=None): class InvalidKeyError(NodeportsException): """Accessed key does not exist""" - def __init__(self, item_key, msg=None): + def __init__(self, item_key: str, msg: Optional[str] = None): super().__init__(f"No port bound with key {item_key}") self.item_key = item_key @@ -46,9 +35,10 @@ def __init__(self, item_key, msg=None): class InvalidItemTypeError(NodeportsException): """Item type incorrect""" - def __init__(self, item_type, item_value): + def __init__(self, item_type: str, item_value: str, msg: Optional[str] = None): super().__init__( - f"Invalid item type, value [{item_value}] does not qualify as type [{item_type}]" + msg + or f"Invalid item type, value [{item_value}] does not qualify as type [{item_type}]" ) self.item_type = item_type self.item_value = item_value @@ -57,7 +47,7 @@ def __init__(self, item_type, item_value): class InvalidProtocolError(NodeportsException): """Invalid protocol used""" - def __init__(self, dct, msg=None): + def __init__(self, dct, msg: Optional[str] = None): super().__init__(f"Invalid protocol used: {dct}\n{msg}") self.dct = dct @@ -73,7 +63,7 @@ class StorageServerIssue(NodeportsException): class S3TransferError(NodeportsException): """S3 transfer error""" - def __init__(self, msg=None): + def __init__(self, msg: Optional[str] = None): super().__init__(msg or "Error while transferring to/from S3 storage") diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py index 0512fa1f47e9..40debd6ac17b 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/filemanager.py @@ -3,18 +3,19 @@ import warnings from contextlib import contextmanager from pathlib import Path -from typing import Optional +from typing import Optional, Tuple -from aiohttp import ClientSession, ClientTimeout +import aiofiles +from aiohttp import ClientPayloadError, ClientSession, ClientTimeout from yarl import URL -import aiofiles +from models_library.settings.services_common import ServicesCommonSettings from simcore_service_storage_sdk import ApiClient, Configuration, UsersApi from simcore_service_storage_sdk.rest import ApiException -from models_library.settings.services_common import ServicesCommonSettings +from tqdm import tqdm -from . import config, exceptions from ..config.http_clients import client_request_settings +from . import config, exceptions log = logging.getLogger(__name__) @@ -87,8 +88,6 @@ async def _get_location_id_from_location_name(store: str, api: UsersApi): raise exceptions.S3InvalidStore(store) except ApiException as err: _handle_api_exception(store, err) - if resp.error: - raise exceptions.StorageConnectionError(store, resp.error.to_str()) async def _get_link(store_id: int, file_id: str, apifct) -> URL: @@ -132,32 +131,55 @@ async def _download_link_to_file(session: ClientSession, url: URL, file_path: Pa if response.status > 299: raise exceptions.TransferError(url) file_path.parent.mkdir(parents=True, exist_ok=True) - async with aiofiles.open(file_path, "wb") as file_pointer: - # await file_pointer.write(await response.read()) - chunk = await response.content.read(CHUNK_SIZE) - while chunk: - await file_pointer.write(chunk) - chunk = await response.content.read(CHUNK_SIZE) - log.debug("Download complete") - return await response.release() - - -async def _file_sender(file_path: Path): - async with aiofiles.open(file_path, "rb") as file_pointer: - chunk = await file_pointer.read(CHUNK_SIZE) - while chunk: - yield chunk - chunk = await file_pointer.read(CHUNK_SIZE) - - -async def _upload_file_to_link(session: ClientSession, url: URL, file_path: Path): + file_size = int(response.headers.get("content-length", 0)) or None + try: + with tqdm( + desc=f"downloading {file_path} [{file_size} bytes]", + total=file_size, + unit="byte", + unit_scale=True, + ) as pbar: + async with aiofiles.open(file_path, "wb") as file_pointer: + chunk = await response.content.read(CHUNK_SIZE) + while chunk: + await file_pointer.write(chunk) + pbar.update(len(chunk)) + chunk = await response.content.read(CHUNK_SIZE) + log.debug("Download complete") + except ClientPayloadError as exc: + raise exceptions.TransferError(url) from exc + + +ETag = str + + +async def _upload_file_to_link( + session: ClientSession, url: URL, file_path: Path +) -> Optional[ETag]: log.debug("Uploading from %s to %s", file_path, url) - async with session.put(url, data=file_path.open("rb")) as resp: - if resp.status > 299: - response_text = await resp.text() - raise exceptions.S3TransferError( - "Could not upload file {}:{}".format(file_path, response_text) - ) + file_size = file_path.stat().st_size + with tqdm( + desc=f"uploading {file_path} [{file_size} bytes]", + total=file_size, + unit="byte", + unit_scale=True, + ) as pbar: + async with session.put(url, data=file_path.open("rb")) as resp: + if resp.status > 299: + response_text = await resp.text() + raise exceptions.S3TransferError( + "Could not upload file {}:{}".format(file_path, response_text) + ) + if resp.status != 200: + response_text = await resp.text() + raise exceptions.S3TransferError( + "Issue when uploading file {}:{}".format(file_path, response_text) + ) + pbar.update(file_size) + # get the S3 etag from the headers + e_tag = resp.headers.get("Etag", None) + log.debug("Uploaded %s to %s, received Etag %s", file_path, url, e_tag) + return e_tag async def download_file_from_s3( @@ -166,7 +188,7 @@ async def download_file_from_s3( store_id: str = None, s3_object: str, local_folder: Path, - session: Optional[ClientSession] = None + session: Optional[ClientSession] = None, ) -> Path: """Downloads a file from S3 @@ -223,12 +245,12 @@ async def download_file_from_link( async def upload_file( *, - store_id: str = None, - store_name: str = None, + store_id: Optional[str] = None, + store_name: Optional[str] = None, s3_object: str, local_file_path: Path, - session: Optional[ClientSession] = None -) -> str: + session: Optional[ClientSession] = None, +) -> Tuple[str, str]: """Uploads a file to S3 :param session: add app[APP_CLIENT_SESSION_KEY] session here otherwise default is opened/closed every call @@ -251,14 +273,14 @@ async def upload_file( if store_name is not None: store_id = await _get_location_id_from_location_name(store_name, api) - upload_link = await _get_upload_link(store_id, s3_object, api) + upload_link: URL = await _get_upload_link(store_id, s3_object, api) if upload_link: - upload_link = URL(upload_link) - + # FIXME: This client should be kept with the nodeports instead of creating one each time async with ClientSessionContextManager(session) as active_session: - await _upload_file_to_link(active_session, upload_link, local_file_path) - - return store_id + e_tag = await _upload_file_to_link( + active_session, upload_link, local_file_path + ) + return store_id, e_tag raise exceptions.S3InvalidPathError(s3_object) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py index 5cc21371c5f9..223a95d94d63 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/nodeports.py @@ -3,6 +3,11 @@ """ import logging + +# pylint: disable=missing-docstring +# pylint: disable=too-many-instance-attributes +# pylint: disable=too-many-arguments +import warnings from pathlib import Path from typing import Optional @@ -13,36 +18,26 @@ log = logging.getLogger(__name__) -# pylint: disable=missing-docstring -# pylint: disable=too-many-instance-attributes -# pylint: disable=too-many-arguments - class Nodeports: - """Allows the client to access the inputs and outputs assigned to the node - - """ - - _version = "0.1" + """Allows the client to access the inputs and outputs assigned to the node""" def __init__( self, - version: str, input_schemas: SchemaItemsList = None, output_schemas: SchemaItemsList = None, input_payloads: DataItemsList = None, outputs_payloads: DataItemsList = None, ): - + warnings.warn( + "node_ports is deprecated, use node_ports_v2 instead", + category=DeprecationWarning, + ) log.debug( - "Initialising Nodeports object with version %s, inputs %s and outputs %s", - version, + "Initialising Nodeports object with inputs %s and outputs %s", input_payloads, outputs_payloads, ) - if self._version != version: - raise exceptions.WrongProtocolVersionError(self._version, version) - if not input_schemas: input_schemas = SchemaItemsList() if not output_schemas: @@ -61,8 +56,7 @@ def __init__( self.autowrite = False log.debug( - "Initialised Nodeports object with version %s, inputs %s and outputs %s", - version, + "Initialised Nodeports object with inputs %s and outputs %s", input_payloads, outputs_payloads, ) @@ -173,6 +167,10 @@ async def _get_node_from_node_uuid(self, node_uuid: str): async def ports(db_manager: Optional[dbmanager.DBManager] = None) -> Nodeports: + warnings.warn( + "node_ports is deprecated, use node_ports_v2 instead", + category=DeprecationWarning, + ) # FIXME: warning every dbmanager create a new db engine! if db_manager is None: # NOTE: keeps backwards compatibility db_manager = dbmanager.DBManager() diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py b/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py index 1033e9e730f5..1624d7db7816 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports/serialization.py @@ -19,7 +19,7 @@ async def create_from_json( db_mgr: DBManager, auto_read: bool = False, auto_write: bool = False ): - """ creates a Nodeports object provided a json configuration in form of a callback function + """creates a Nodeports object provided a json configuration in form of a callback function :param db_mgr: interface object to connect to nodeports description :param auto_read: the nodeports object shall automatically update itself when set to True, defaults to False @@ -64,7 +64,7 @@ async def create_nodeports_from_uuid(db_mgr: DBManager, node_uuid: str): async def save_to_json(nodeports_obj) -> None: - """ Encodes a Nodeports object to json and calls a linked writer if available. + """Encodes a Nodeports object to json and calls a linked writer if available. :param nodeports_obj: the object to encode :type nodeports_obj: Nodeports @@ -93,8 +93,10 @@ def default(self, o): # pylint: disable=E0202 log.debug("Encoding Nodeports object") return { # pylint: disable=W0212 - "version": o._version, - "schema": {"inputs": o._input_schemas, "outputs": o._output_schemas,}, + "schema": { + "inputs": o._input_schemas, + "outputs": o._output_schemas, + }, "inputs": o._inputs_payloads, "outputs": o._outputs_payloads, } @@ -139,7 +141,6 @@ def __decodeNodePorts(dct: Dict): ) return nodeports.Nodeports( - dct["version"], SchemaItemsList(decoded_input_schema), SchemaItemsList(decoded_output_schema), DataItemsList(decoded_input_payload), diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py new file mode 100644 index 000000000000..a64f9640b1aa --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py @@ -0,0 +1,24 @@ +import logging +from typing import Optional + +from ..node_ports import config as node_config +from ..node_ports import exceptions +from ..node_ports.dbmanager import DBManager +from .nodeports_v2 import Nodeports +from .port import Port +from .serialization_v2 import load + +log = logging.getLogger(__name__) + + +async def ports(db_manager: Optional[DBManager] = None) -> Nodeports: + log.debug("creating node_ports_v2 object using provided dbmanager: %s", db_manager) + # FIXME: warning every dbmanager create a new db engine! + if db_manager is None: # NOTE: keeps backwards compatibility + log.debug("no db manager provided, creating one...") + db_manager = DBManager() + + return await load(db_manager, node_config.NODE_UUID, auto_update=True) + + +__all__ = ["ports", "node_config", "exceptions", "Port"] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py new file mode 100644 index 000000000000..572ba9f210f9 --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/links.py @@ -0,0 +1,26 @@ +from pathlib import Path +from typing import Union + +from models_library.projects_nodes_io import UUID_REGEX, BaseFileLink, DownloadLink +from models_library.projects_nodes_io import PortLink as BasePortLink +from pydantic import Extra, Field, StrictBool, StrictFloat, StrictInt, StrictStr + + +class PortLink(BasePortLink): + node_uuid: str = Field(..., regex=UUID_REGEX, alias="nodeUuid") + + +class FileLink(BaseFileLink): + """ allow all kind of file links """ + + class Config: + extra = Extra.allow + + +DataItemValue = Union[ + StrictBool, StrictInt, StrictFloat, StrictStr, DownloadLink, PortLink, FileLink +] + +ItemConcreteValue = Union[int, float, bool, str, Path] + +__all__ = ["FileLink", "DownloadLink", "PortLink", "DataItemValue", "ItemConcreteValue"] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py new file mode 100644 index 000000000000..a4da941a195b --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -0,0 +1,91 @@ +import logging +from pathlib import Path +from typing import Any, Callable, Coroutine, Type + +from pydantic import BaseModel, Field + +from ..node_ports.dbmanager import DBManager +from ..node_ports.exceptions import PortNotFound, UnboundPortError +from .links import ItemConcreteValue +from .port_utils import is_file_type +from .ports_mapping import InputsList, OutputsList + +log = logging.getLogger(__name__) + + +class Nodeports(BaseModel): + internal_inputs: InputsList = Field(..., alias="inputs") + internal_outputs: OutputsList = Field(..., alias="outputs") + db_manager: DBManager + node_uuid: str + save_to_db_cb: Callable[["Nodeports"], Coroutine[Any, Any, None]] + node_port_creator_cb: Callable[[DBManager, str], Coroutine[Any, Any, "Nodeports"]] + auto_update: bool = False + + class Config: + arbitrary_types_allowed = True + + def __init__(self, **data: Any): + super().__init__(**data) + # let's pass ourselves down + for input_key in self.internal_inputs: + self.internal_inputs[input_key]._node_ports = self + for output_key in self.internal_outputs: + self.internal_outputs[output_key]._node_ports = self + + @property + async def inputs(self) -> InputsList: + log.debug("Getting inputs with autoupdate: %s", self.auto_update) + if self.auto_update: + await self._auto_update_from_db() + return self.internal_inputs + + @property + async def outputs(self) -> OutputsList: + log.debug("Getting outputs with autoupdate: %s", self.auto_update) + if self.auto_update: + await self._auto_update_from_db() + return self.internal_outputs + + async def get(self, item_key: str) -> ItemConcreteValue: + try: + return await (await self.inputs)[item_key].get() + except UnboundPortError: + # not available try outputs + pass + # if this fails it will raise an exception + return await (await self.outputs)[item_key].get() + + async def set(self, item_key: str, item_value: ItemConcreteValue) -> None: + try: + await (await self.inputs)[item_key].set(item_value) + return + except UnboundPortError: + # not available try outputs + pass + # if this fails it will raise an exception + await (await self.outputs)[item_key].set(item_value) + + async def set_file_by_keymap(self, item_value: Path) -> None: + for output in (await self.outputs).values(): + if is_file_type(output.property_type) and output.file_to_key_map: + if item_value.name in output.file_to_key_map: + await output.set(item_value) + return + raise PortNotFound(msg=f"output port for item {item_value} not found") + + async def _node_ports_creator_cb(self, node_uuid: str) -> Type["Nodeports"]: + return await self.node_port_creator_cb(self.db_manager, node_uuid) + + async def _auto_update_from_db(self) -> None: + # get the newest from the DB + updated_node_ports = await self._node_ports_creator_cb(self.node_uuid) + # update our stuff + self.internal_inputs = updated_node_ports.internal_inputs + self.internal_outputs = updated_node_ports.internal_outputs + # let's pass ourselves down + # pylint: disable=protected-access + for input_key in self.internal_inputs: + self.internal_inputs[input_key]._node_ports = self + for output_key in self.internal_outputs: + self.internal_outputs[output_key]._node_ports = self diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py new file mode 100644 index 000000000000..a7d25467974d --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -0,0 +1,123 @@ +import logging +from pathlib import Path +from pprint import pformat +from typing import Any, Dict, Optional, Tuple, Type + +from models_library.services import PROPERTY_KEY_RE, ServiceProperty +from pydantic import Field, PrivateAttr, validator + +from ..node_ports.exceptions import InvalidItemTypeError +from . import port_utils +from .links import DataItemValue, DownloadLink, FileLink, ItemConcreteValue, PortLink + +log = logging.getLogger(__name__) + + +TYPE_TO_PYTYPE: Dict[str, Type[ItemConcreteValue]] = { + "integer": int, + "number": float, + "boolean": bool, + "string": str, +} + + +class Port(ServiceProperty): + key: str = Field(..., regex=PROPERTY_KEY_RE) + widget: Optional[Dict[str, Any]] = None + + value: Optional[DataItemValue] + + _py_value_type: Tuple[Type[ItemConcreteValue]] = PrivateAttr() + _py_value_converter: Type[ItemConcreteValue] = PrivateAttr() + _node_ports = PrivateAttr() + _used_default_value: bool = PrivateAttr(False) + + @validator("value", always=True) + @classmethod + def ensure_value(cls, v: DataItemValue, values: Dict[str, Any]) -> DataItemValue: + if "property_type" in values and port_utils.is_file_type( + values["property_type"] + ): + if v is not None and not isinstance(v, (FileLink, DownloadLink, PortLink)): + raise ValueError( + f"[{values['property_type']}] must follow {FileLink.schema()}, {DownloadLink.schema()} or {PortLink.schema()}" + ) + return v + + def __init__(self, **data: Any): + super().__init__(**data) + + self._py_value_type = ( + (Path, str) + if port_utils.is_file_type(self.property_type) + else (TYPE_TO_PYTYPE[self.property_type]) + ) + self._py_value_converter = ( + Path + if port_utils.is_file_type(self.property_type) + else TYPE_TO_PYTYPE[self.property_type] + ) + if ( + self.value is None + and self.default_value is not None + and not port_utils.is_file_type(self.property_type) + ): + self.value = self.default_value + self._used_default_value = True + + async def get(self) -> ItemConcreteValue: + log.debug( + "getting %s[%s] with value %s", + self.key, + self.property_type, + pformat(self.value), + ) + + if self.value is None: + return None + + value = None + if isinstance(self.value, PortLink): + # this is a link to another node + value = await port_utils.get_value_from_link( + # pylint: disable=protected-access + self.key, + self.value, + self.file_to_key_map, + self._node_ports._node_ports_creator_cb, + ) + elif isinstance(self.value, FileLink): + # this is a link from storage + value = await port_utils.pull_file_from_store( + self.key, self.file_to_key_map, self.value + ) + elif isinstance(self.value, DownloadLink): + # this is a downloadable link + value = await port_utils.pull_file_from_download_link( + self.key, self.file_to_key_map, self.value + ) + else: + # this is directly the value + value = self.value + + return self._py_value_converter(value) + + async def set(self, new_value: ItemConcreteValue) -> None: + log.debug( + "setting %s[%s] with value %s", self.key, self.property_type, new_value + ) + final_value: Optional[DataItemValue] = None + if new_value is not None: + # convert the concrete value to a data value + converted_value: ItemConcreteValue = self._py_value_converter(new_value) + + if isinstance(converted_value, Path): + if not converted_value.exists() or not converted_value.is_file(): + raise InvalidItemTypeError(self.property_type, str(new_value)) + final_value = await port_utils.push_file_to_store(converted_value) + else: + final_value = converted_value + + self.value = final_value + self._used_default_value = False + await self._node_ports.save_to_db_cb(self._node_ports) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py new file mode 100644 index 000000000000..0b72695d2af7 --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -0,0 +1,106 @@ +import logging +import shutil +from pathlib import Path +from typing import Any, Callable, Coroutine, Dict, Optional + +from ..node_ports import config, data_items_utils, filemanager +from .links import DownloadLink, FileLink, ItemConcreteValue, PortLink + +log = logging.getLogger(__name__) + + +async def get_value_from_link( + key: str, + value: PortLink, + fileToKeyMap: Optional[Dict[str, str]], + node_port_creator: Callable[[str], Coroutine[Any, Any, Any]], +) -> ItemConcreteValue: + log.debug("Getting value %s", value) + # create a node ports for the other node + other_nodeports = await node_port_creator(value.node_uuid) + # get the port value through that guy + log.debug("Received node from DB %s, now returning value", other_nodeports) + + value = await other_nodeports.get(value.output) + if isinstance(value, Path): + file_name = value.name + # move the file to the right final location + # if a file alias is present use it + if fileToKeyMap: + file_name = next(iter(fileToKeyMap)) + + file_path = data_items_utils.create_file_path(key, file_name) + if value == file_path: + # this is a corner case: in case the output key of the other node has the same name as the input key + return value + if file_path.exists(): + file_path.unlink() + file_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(value), str(file_path)) + value = file_path + + return value + + +async def pull_file_from_store( + key: str, fileToKeyMap: Optional[Dict[str, str]], value: FileLink +) -> Path: + log.debug("Getting value from storage %s", value) + # do not make any assumption about s3_path, it is a str containing stuff that can be anything depending on the store + local_path = data_items_utils.create_folder_path(key) + downloaded_file = await filemanager.download_file_from_s3( + store_id=value.store, s3_object=value.path, local_folder=local_path + ) + # if a file alias is present use it to rename the file accordingly + if fileToKeyMap: + renamed_file = local_path / next(iter(fileToKeyMap)) + if downloaded_file != renamed_file: + if renamed_file.exists(): + renamed_file.unlink() + shutil.move(downloaded_file, renamed_file) + downloaded_file = renamed_file + + return downloaded_file + + +async def push_file_to_store(file: Path) -> FileLink: + log.debug("file path %s will be uploaded to s3", file) + s3_object = data_items_utils.encode_file_id( + file, project_id=config.PROJECT_ID, node_id=config.NODE_UUID + ) + store_id, e_tag = await filemanager.upload_file( + store_name=config.STORE, s3_object=s3_object, local_file_path=file + ) + log.debug("file path %s uploaded, received ETag %s", file, e_tag) + return FileLink(store=store_id, path=s3_object, e_tag=e_tag) + + +async def pull_file_from_download_link( + key: str, fileToKeyMap: Optional[Dict[str, str]], value: DownloadLink +) -> Path: + log.debug( + "Getting value from download link [%s] with label %s", + value.download_link, + value.label, + ) + + local_path = data_items_utils.create_folder_path(key) + downloaded_file = await filemanager.download_file_from_link( + value.download_link, + local_path, + ) + + # if a file alias is present use it to rename the file accordingly + if fileToKeyMap: + renamed_file = local_path / next(iter(fileToKeyMap)) + if downloaded_file != renamed_file: + if renamed_file.exists(): + renamed_file.unlink() + shutil.move(downloaded_file, renamed_file) + downloaded_file = renamed_file + + return downloaded_file + + +def is_file_type(port_type: str) -> bool: + return port_type.startswith("data:") diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py new file mode 100644 index 000000000000..2afa839e4aaa --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/ports_mapping.py @@ -0,0 +1,44 @@ +from typing import Dict, ItemsView, Iterator, KeysView, Union, ValuesView + +from models_library.services import PROPERTY_KEY_RE +from pydantic import BaseModel, constr + +from ..node_ports.exceptions import UnboundPortError +from .port import Port + +PortKey: constr = constr(regex=PROPERTY_KEY_RE) + + +class PortsMapping(BaseModel): + __root__: Dict[PortKey, Port] + + def __getitem__(self, key: Union[int, PortKey]) -> Port: + if isinstance(key, int): + if key < len(self.__root__): + key = list(self.__root__.keys())[key] + if not key in self.__root__: + raise UnboundPortError(key) + return self.__root__[key] + + def __iter__(self) -> Iterator[PortKey]: + return iter(self.__root__) + + def keys(self) -> KeysView[PortKey]: + return self.__root__.keys() + + def items(self) -> ItemsView[PortKey, Port]: + return self.__root__.items() + + def values(self) -> ValuesView[Port]: + return self.__root__.values() + + def __len__(self) -> int: + return self.__root__.__len__() + + +class InputsList(PortsMapping): + __root__: Dict[PortKey, Port] + + +class OutputsList(PortsMapping): + __root__: Dict[PortKey, Port] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py new file mode 100644 index 000000000000..2e2d05c4a96a --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -0,0 +1,103 @@ +import json +import logging +from pprint import pformat +from typing import Any, Dict, Set + +from aiopg.sa.result import RowProxy + +from ..node_ports.dbmanager import DBManager +from ..node_ports.exceptions import InvalidProtocolError +from .nodeports_v2 import Nodeports + +log = logging.getLogger(__name__) + +NODE_REQUIRED_KEYS: Set[str] = { + "schema", + "inputs", + "outputs", +} + + +async def load( + db_manager: DBManager, node_uuid: str, auto_update: bool = False +) -> Nodeports: + """creates a nodeport object from a row from comp_tasks""" + log.debug( + "creating node_ports_v2 object from node %s with auto_uptate %s", + node_uuid, + auto_update, + ) + row: RowProxy = await db_manager.get_ports_configuration_from_node_uuid(node_uuid) + port_cfg = json.loads(row) + if any(k not in port_cfg for k in NODE_REQUIRED_KEYS): + raise InvalidProtocolError( + port_cfg, "nodeport in comp_task does not follow protocol" + ) + # convert to our internal node ports + _PY_INT = "__root__" + node_ports_cfg: Dict[str, Dict[str, Any]] = { + "inputs": {_PY_INT: {}}, + "outputs": {_PY_INT: {}}, + } + for port_type in ["inputs", "outputs"]: + # schemas first + node_ports_cfg.update( + { + port_type: {_PY_INT: port_cfg["schema"][port_type]}, + } + ) + # add the key and the payload + for key, port_value in node_ports_cfg[port_type][_PY_INT].items(): + port_value["key"] = key + port_value["value"] = port_cfg[port_type].get(key, None) + + ports = Nodeports( + **node_ports_cfg, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=dump, + node_port_creator_cb=load, + auto_update=auto_update, + ) + log.debug( + "created node_ports_v2 object %s", + pformat(ports, indent=2), + ) + return ports + + +async def dump(nodeports: Nodeports) -> None: + log.debug( + "dumping node_ports_v2 object %s", + pformat(nodeports, indent=2), + ) + _nodeports_cfg = nodeports.dict( + include={"internal_inputs", "internal_outputs"}, + by_alias=True, + exclude_unset=True, + ) + + # convert to DB + port_cfg = {"schema": {"inputs": {}, "outputs": {}}, "inputs": {}, "outputs": {}} + for port_type in ["inputs", "outputs"]: + for port_key, port_values in _nodeports_cfg[port_type].items(): + # schemas + key_schema = { + k: v + for k, v in _nodeports_cfg[port_type][port_key].items() + if k not in ["key", "value"] + } + port_cfg["schema"][port_type][port_key] = key_schema + # payload (only if default value was not used) + # pylint: disable=protected-access + if ( + port_values["value"] is not None + and not getattr(nodeports, f"internal_{port_type}")[ + port_key + ]._used_default_value + ): + port_cfg[port_type][port_key] = port_values["value"] + + await nodeports.db_manager.write_ports_configuration( + json.dumps(port_cfg), nodeports.node_uuid + ) diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index 7b105bf72a08..bc2461990ec6 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -1,9 +1,14 @@ +import json + # pylint:disable=unused-argument # pylint:disable=redefined-outer-name import sys from pathlib import Path +from typing import Any, Dict import pytest +import simcore_sdk +from simcore_sdk.node_ports import config as node_config ## HELPERS current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent @@ -17,9 +22,17 @@ "pytest_simcore.postgres_service", "pytest_simcore.minio_service", "pytest_simcore.simcore_storage_service", + "pytest_simcore.services_api_mocks_for_aiohttp_clients", ] +@pytest.fixture(scope="session") +def package_dir(): + pdir = Path(simcore_sdk.__file__).resolve().parent + assert pdir.exists() + return pdir + + @pytest.fixture(scope="session") def osparc_simcore_root_dir() -> Path: """ osparc-simcore repo root dir """ @@ -42,3 +55,34 @@ def env_devel_file(osparc_simcore_root_dir) -> Path: env_devel_fpath = osparc_simcore_root_dir / ".env-devel" assert env_devel_fpath.exists() return env_devel_fpath + + +@pytest.fixture(scope="session") +def default_configuration_file() -> Path: + path = current_dir / "mock" / "default_config.json" + assert path.exists() + return path + + +@pytest.fixture(scope="session") +def default_configuration(default_configuration_file: Path) -> Dict[str, Any]: + config = json.loads(default_configuration_file.read_text()) + return config + + +@pytest.fixture(scope="session") +def empty_configuration_file() -> Path: + path = current_dir / "mock" / "empty_config.json" + assert path.exists() + return path + + +@pytest.fixture(scope="module") +def node_ports_config( + postgres_dsn: Dict[str, str], minio_config: Dict[str, str] +) -> None: + node_config.POSTGRES_DB = postgres_dsn["database"] + node_config.POSTGRES_ENDPOINT = f"{postgres_dsn['host']}:{postgres_dsn['port']}" + node_config.POSTGRES_USER = postgres_dsn["user"] + node_config.POSTGRES_PW = postgres_dsn["password"] + node_config.BUCKET = minio_config["bucket_name"] diff --git a/packages/simcore-sdk/tests/helpers/utils_docker.py b/packages/simcore-sdk/tests/helpers/utils_docker.py index 2b0bcb4de339..e5cc50b62071 100644 --- a/packages/simcore-sdk/tests/helpers/utils_docker.py +++ b/packages/simcore-sdk/tests/helpers/utils_docker.py @@ -1,4 +1,3 @@ - import logging import os import subprocess @@ -12,38 +11,50 @@ log = logging.getLogger(__name__) + @retry( - wait=wait_fixed(2), - stop=stop_after_attempt(10), - after=after_log(log, logging.WARN)) -def get_service_published_port(service_name: str, target_port: Optional[int]=None) -> str: + wait=wait_fixed(2), stop=stop_after_attempt(10), after=after_log(log, logging.WARN) +) +def get_service_published_port( + service_name: str, target_port: Optional[int] = None +) -> str: """ - WARNING: ENSURE that service name exposes a port in Dockerfile file or docker-compose config file + WARNING: ENSURE that service name exposes a port in Dockerfile file or docker-compose config file """ # NOTE: retries since services can take some time to start client = docker.from_env() services = [x for x in client.services.list() if service_name in x.name] if not services: - raise RuntimeError(f"Cannot find published port for service '{service_name}'. Probably services still not started.") + raise RuntimeError( + f"Cannot find published port for service '{service_name}'. Probably services still not started." + ) service_ports = services[0].attrs["Endpoint"].get("Ports") if not service_ports: - raise RuntimeError(f"Cannot find published port for service '{service_name}' in endpoint. Probably services still not started.") + raise RuntimeError( + f"Cannot find published port for service '{service_name}' in endpoint. Probably services still not started." + ) published_port = None - msg = ", ".join( f"{p.get('TargetPort')} -> {p.get('PublishedPort')}" for p in service_ports ) + msg = ", ".join( + f"{p.get('TargetPort')} -> {p.get('PublishedPort')}" for p in service_ports + ) if target_port is None: - if len(service_ports)>1: - log.warning("Multiple ports published in service '%s': %s. Defaulting to first", service_name, msg) + if len(service_ports) > 1: + log.warning( + "Multiple ports published in service '%s': %s. Defaulting to first", + service_name, + msg, + ) published_port = service_ports[0]["PublishedPort"] else: target_port = int(target_port) for p in service_ports: - if p['TargetPort'] == target_port: - published_port = p['PublishedPort'] + if p["TargetPort"] == target_port: + published_port = p["PublishedPort"] break if published_port is None: @@ -55,28 +66,37 @@ def get_service_published_port(service_name: str, target_port: Optional[int]=Non def run_docker_compose_config( docker_compose_paths: Union[List[Path], Path], workdir: Path, - destination_path: Optional[Path]=None) -> Dict: - """ Runs docker-compose config to validate and resolve a compose file configuration + destination_path: Optional[Path] = None, +) -> Dict: + """Runs docker-compose config to validate and resolve a compose file configuration - - Composes all configurations passed in 'docker_compose_paths' - - Takes 'workdir' as current working directory (i.e. all '.env' files there will be captured) - - Saves resolved output config to 'destination_path' (if given) + - Composes all configurations passed in 'docker_compose_paths' + - Takes 'workdir' as current working directory (i.e. all '.env' files there will be captured) + - Saves resolved output config to 'destination_path' (if given) """ if not isinstance(docker_compose_paths, List): - docker_compose_paths = [docker_compose_paths, ] + docker_compose_paths = [ + docker_compose_paths, + ] temp_dir = None if destination_path is None: - temp_dir = Path(tempfile.mkdtemp(prefix='')) - destination_path = temp_dir / 'docker-compose.yml' + temp_dir = Path(tempfile.mkdtemp(prefix="")) + destination_path = temp_dir / "docker-compose.yml" - config_paths = [ f"-f {os.path.relpath(docker_compose_path, workdir)}" for docker_compose_path in docker_compose_paths] + config_paths = [ + f"-f {os.path.relpath(docker_compose_path, workdir)}" + for docker_compose_path in docker_compose_paths + ] configs_prefix = " ".join(config_paths) - subprocess.run( f"docker-compose {configs_prefix} config > {destination_path}", - shell=True, check=True, - cwd=workdir) + subprocess.run( + f"docker-compose {configs_prefix} config > {destination_path}", + shell=True, + check=True, + cwd=workdir, + ) with destination_path.open() as f: config = yaml.safe_load(f) diff --git a/packages/simcore-sdk/tests/helpers/utils_environs.py b/packages/simcore-sdk/tests/helpers/utils_environs.py index 006245ef43be..40a54b5206df 100644 --- a/packages/simcore-sdk/tests/helpers/utils_environs.py +++ b/packages/simcore-sdk/tests/helpers/utils_environs.py @@ -8,15 +8,16 @@ import yaml -VARIABLE_SUBSTITUTION = re.compile(r'\$\{(\w+)+') # +VARIABLE_SUBSTITUTION = re.compile(r"\$\{(\w+)+") # + def load_env(file_handler) -> Dict: - """ Deserializes an environment file like .env-devel and - returns a key-value map of the environment + """Deserializes an environment file like .env-devel and + returns a key-value map of the environment - Analogous to json.load + Analogous to json.load """ - PATTERN_ENVIRON_EQUAL= re.compile(r"^(\w+)=(.*)$") + PATTERN_ENVIRON_EQUAL = re.compile(r"^(\w+)=(.*)$") # Works even for `POSTGRES_EXPORTER_DATA_SOURCE_NAME=postgresql://simcore:simcore@postgres:5432/simcoredb?sslmode=disable` environ = {} @@ -27,30 +28,41 @@ def load_env(file_handler) -> Dict: environ[key] = str(value) return environ -def eval_environs_in_docker_compose(docker_compose: Dict, docker_compose_dir: Path, - host_environ: Dict=None, *, use_env_devel=True): - """ Resolves environments in docker compose and sets them under 'environment' section - TODO: deprecated. Use instead docker-compose config in services/web/server/tests/integration/fixtures/docker_compose.py - SEE https://docs.docker.com/compose/environment-variables/ +def eval_environs_in_docker_compose( + docker_compose: Dict, + docker_compose_dir: Path, + host_environ: Dict = None, + *, + use_env_devel=True +): + """Resolves environments in docker compose and sets them under 'environment' section + + TODO: deprecated. Use instead docker-compose config in services/web/server/tests/integration/fixtures/docker_compose.py + SEE https://docs.docker.com/compose/environment-variables/ """ content = deepcopy(docker_compose) for _name, service in content["services"].items(): - replace_environs_in_docker_compose_service(service, docker_compose_dir, - host_environ, use_env_devel=use_env_devel) + replace_environs_in_docker_compose_service( + service, docker_compose_dir, host_environ, use_env_devel=use_env_devel + ) return content -def replace_environs_in_docker_compose_service(service_section: Dict, + +def replace_environs_in_docker_compose_service( + service_section: Dict, docker_compose_dir: Path, - host_environ: Dict=None, - *, use_env_devel=True): - """ Resolves environments in docker-compose's service section, - drops any reference to env_file and sets all - environs 'environment' section + host_environ: Dict = None, + *, + use_env_devel=True +): + """Resolves environments in docker-compose's service section, + drops any reference to env_file and sets all + environs 'environment' section - NOTE: service_section gets modified! + NOTE: service_section gets modified! - SEE https://docs.docker.com/compose/environment-variables/ + SEE https://docs.docker.com/compose/environment-variables/ """ service_environ = {} @@ -77,16 +89,23 @@ def replace_environs_in_docker_compose_service(service_section: Dict, # In VAR=${FOO} matches VAR and FOO # - TODO: add to read defaults envkey = m.groups()[0] - value = host_environ[envkey] # fails when variable in docker-compose is NOT defined + value = host_environ[ + envkey + ] # fails when variable in docker-compose is NOT defined service_environ[key] = value service_section["environment"] = service_environ -def eval_service_environ(docker_compose_path:Path, service_name:str, - host_environ: Dict=None, - image_environ: Dict=None, - *, use_env_devel=True) -> Dict: - """ Deduces a service environment with it runs in a stack from confirmation + +def eval_service_environ( + docker_compose_path: Path, + service_name: str, + host_environ: Dict = None, + image_environ: Dict = None, + *, + use_env_devel=True +) -> Dict: + """Deduces a service environment with it runs in a stack from confirmation :param docker_compose_path: path to stack configuration :type docker_compose_path: Path @@ -104,8 +123,9 @@ def eval_service_environ(docker_compose_path:Path, service_name:str, content = yaml.safe_load(f) service = content["services"][service_name] - replace_environs_in_docker_compose_service(service, docker_compose_dir, - host_environ, use_env_devel=use_env_devel) + replace_environs_in_docker_compose_service( + service, docker_compose_dir, host_environ, use_env_devel=use_env_devel + ) host_environ = host_environ or {} image_environ = image_environ or {} diff --git a/packages/simcore-sdk/tests/helpers/utils_port_v2.py b/packages/simcore-sdk/tests/helpers/utils_port_v2.py new file mode 100644 index 000000000000..19a8957668e5 --- /dev/null +++ b/packages/simcore-sdk/tests/helpers/utils_port_v2.py @@ -0,0 +1,46 @@ +from typing import Any, Dict, Optional, Type, Union + +from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList + + +def create_valid_port_config(conf_type: str, **kwargs) -> Dict[str, Any]: + valid_config = { + "key": f"some_{conf_type}", + "label": "some label", + "description": "some description", + "type": conf_type, + "displayOrder": 2.3, + } + valid_config.update(kwargs) + return valid_config + + +def create_valid_port_mapping( + mapping_class: Type[Union[InputsList, OutputsList]], + suffix: str, + file_to_key: Optional[str] = None, +) -> Type[Union[InputsList, OutputsList]]: + port_cfgs: Dict[str, Any] = {} + for t, v in { + "integer": 43, + "number": 45.6, + "boolean": True, + "string": "dfgjkhdf", + }.items(): + port = create_valid_port_config( + t, + key=f"{'input' if mapping_class==InputsList else 'output'}_{t}_{suffix}", + value=v, + ) + port_cfgs[port["key"]] = port + + key_for_file_port = ( + f"{'input' if mapping_class==InputsList else 'output'}_file_{suffix}" + ) + port_cfgs[key_for_file_port] = create_valid_port_config( + "data:*/*", + key=key_for_file_port, + fileToKeyMap={file_to_key: key_for_file_port} if file_to_key else None, + ) + port_mapping = mapping_class(**{"__root__": port_cfgs}) + return port_mapping diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index a17fdfb30cef..618efc4b206a 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -3,43 +3,39 @@ # pylint:disable=redefined-outer-name # pylint:disable=too-many-arguments +import asyncio import json import sys import uuid from pathlib import Path -from typing import Any, Dict, List, Tuple +from typing import Any, Callable, Dict, List, Tuple +import np_helpers import pytest import sqlalchemy as sa -from yarl import URL - -import np_helpers -from simcore_sdk.models.pipeline_models import ( - ComputationalPipeline, - ComputationalTask, -) +from simcore_sdk.models.pipeline_models import ComputationalPipeline, ComputationalTask from simcore_sdk.node_ports import node_config +from yarl import URL current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent -@pytest.fixture -def nodeports_config( - postgres_dsn: Dict[str, str], minio_config: Dict[str, str] -) -> None: - node_config.POSTGRES_DB = postgres_dsn["database"] - node_config.POSTGRES_ENDPOINT = f"{postgres_dsn['host']}:{postgres_dsn['port']}" - node_config.POSTGRES_USER = postgres_dsn["user"] - node_config.POSTGRES_PW = postgres_dsn["password"] - node_config.BUCKET = minio_config["bucket_name"] - - @pytest.fixture def user_id() -> int: # see fixtures/postgres.py yield 1258 +@pytest.fixture +def project_id() -> str: + return str(uuid.uuid4()) + + +@pytest.fixture +def node_uuid() -> str: + return str(uuid.uuid4()) + + @pytest.fixture def s3_simcore_location() -> str: yield np_helpers.SIMCORE_STORE @@ -47,13 +43,13 @@ def s3_simcore_location() -> str: @pytest.fixture async def filemanager_cfg( - loop, + loop: asyncio.events.AbstractEventLoop, storage_service: URL, devel_environ: Dict, user_id: str, bucket: str, postgres_db, # waits for db and initializes it -): +) -> None: node_config.STORAGE_ENDPOINT = f"{storage_service.host}:{storage_service.port}" node_config.USER_ID = user_id node_config.BUCKET = bucket @@ -61,17 +57,7 @@ async def filemanager_cfg( @pytest.fixture -def project_id() -> str: - return str(uuid.uuid4()) - - -@pytest.fixture -def node_uuid() -> str: - return str(uuid.uuid4()) - - -@pytest.fixture -def file_uuid(project_id, node_uuid) -> str: +def file_uuid(project_id: str, node_uuid: str) -> Callable: def create(file_path: Path, project: str = None, node: str = None): if project is None: project = project_id @@ -82,25 +68,15 @@ def create(file_path: Path, project: str = None, node: str = None): yield create -@pytest.fixture -def default_configuration_file() -> Path: - return current_dir / "mock" / "default_config.json" - - -@pytest.fixture -def empty_configuration_file() -> Path: - return current_dir / "mock" / "empty_config.json" - - @pytest.fixture() def default_configuration( - nodeports_config, + node_ports_config, bucket, postgres_session: sa.orm.session.Session, default_configuration_file: Path, project_id, node_uuid, -): +) -> Dict: # prepare database with default configuration json_configuration = default_configuration_file.read_text() @@ -117,16 +93,18 @@ def default_configuration( @pytest.fixture() -def node_link(): - def create_node_link(key: str): +def node_link() -> Callable: + def create_node_link(key: str) -> Dict[str, str]: return {"nodeUuid": "TEST_NODE_UUID", "output": key} yield create_node_link @pytest.fixture() -def store_link(minio_service, bucket, file_uuid, s3_simcore_location): - def create_store_link(file_path: Path, project_id: str = None, node_id: str = None): +def store_link(minio_service, bucket, file_uuid, s3_simcore_location) -> Callable: + def create_store_link( + file_path: Path, project_id: str = None, node_id: str = None + ) -> Dict[str, str]: # upload the file to S3 assert Path(file_path).exists() file_id = file_uuid(file_path, project_id, node_id) @@ -141,19 +119,19 @@ def create_store_link(file_path: Path, project_id: str = None, node_id: str = No @pytest.fixture(scope="function") def special_configuration( - nodeports_config, - bucket, + node_ports_config: None, + bucket: str, postgres_session: sa.orm.session.Session, empty_configuration_file: Path, - project_id, - node_uuid, -): + project_id: str, + node_uuid: str, +) -> Callable: def create_config( inputs: List[Tuple[str, str, Any]] = None, outputs: List[Tuple[str, str, Any]] = None, project_id: str = project_id, node_id: str = node_uuid, - ): + ) -> Tuple[Dict, str, str]: config_dict = json.loads(empty_configuration_file.read_text()) _assign_config(config_dict, "inputs", inputs) _assign_config(config_dict, "outputs", outputs) @@ -174,12 +152,12 @@ def create_config( @pytest.fixture(scope="function") def special_2nodes_configuration( - nodeports_config, - bucket, + node_ports_config: None, + bucket: str, postgres_session: sa.orm.session.Session, empty_configuration_file: Path, - project_id, - node_uuid, + project_id: str, + node_uuid: str, ): def create_config( prev_node_inputs: List[Tuple[str, str, Any]] = None, @@ -189,7 +167,7 @@ def create_config( project_id: str = project_id, previous_node_id: str = node_uuid, node_id: str = "asdasdadsa", - ): + ) -> Tuple[Dict, str, str]: _create_new_pipeline(postgres_session, project_id) # create previous node @@ -225,15 +203,20 @@ def create_config( postgres_session.commit() -def _create_new_pipeline(session, project: str) -> str: +def _create_new_pipeline(postgres_session: sa.orm.session.Session, project: str) -> str: # pylint: disable=no-member new_Pipeline = ComputationalPipeline(project_id=project) - session.add(new_Pipeline) - session.commit() + postgres_session.add(new_Pipeline) + postgres_session.commit() return new_Pipeline.project_id -def _set_configuration(session, project_id: str, node_id: str, json_configuration: str): +def _set_configuration( + postgres_session: sa.orm.session.Session, + project_id: str, + node_id: str, + json_configuration: str, +) -> str: node_uuid = node_id json_configuration = json_configuration.replace("SIMCORE_NODE_UUID", str(node_uuid)) configuration = json.loads(json_configuration) @@ -245,8 +228,8 @@ def _set_configuration(session, project_id: str, node_id: str, json_configuratio inputs=configuration["inputs"], outputs=configuration["outputs"], ) - session.add(new_Node) - session.commit() + postgres_session.add(new_Node) + postgres_session.commit() return node_uuid diff --git a/packages/simcore-sdk/tests/integration/mock/default_config.json b/packages/simcore-sdk/tests/integration/mock/default_config.json deleted file mode 100644 index b0508a717308..000000000000 --- a/packages/simcore-sdk/tests/integration/mock/default_config.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "version":"0.1", - "schema": { - "inputs": { - "in_1":{ - "displayOrder": 0, - "label": "computational data", - "description": "these are computed data out of a pipeline", - "type": "data:*/*", - "defaultValue": null, - "fileToKeyMap": { - "input1.txt":"in_1" - }, - "widget": null - }, - "in_5":{ - "displayOrder": 2, - "label": "some number", - "description": "numbering things", - "type": "integer", - "defaultValue": 666, - "fileToKeyMap":{}, - "widget": null - } - }, - "outputs" : { - "out_1": { - "displayOrder":0, - "label": "some boolean output", - "description": "could be true or false...", - "type": "boolean", - "defaultValue": null, - "fileToKeyMap":{}, - "widget": null - }, - "out_2": { - "displayOrder":1, - "label": "some file output", - "description": "could be anything...", - "type": "data:*/*", - "defaultValue": null, - "fileToKeyMap":{}, - "widget": null - } - } - }, - "inputs": { - "in_1": { - "nodeUuid":"456465-45ffd", - "output": "outFile" - } - }, - "outputs": { - "out_1": false, - "out_2": { - "store":"z43-s3", - "path": "/simcore/outputControllerOut.dat" - } - } -} \ No newline at end of file diff --git a/packages/simcore-sdk/tests/integration/mock/empty_config.json b/packages/simcore-sdk/tests/integration/mock/empty_config.json deleted file mode 100644 index 943566ea71c2..000000000000 --- a/packages/simcore-sdk/tests/integration/mock/empty_config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version":"0.1", - "schema": { - "inputs": { - }, - "outputs" : { - } - }, - "inputs": { - }, - "outputs": { - } -} \ No newline at end of file diff --git a/packages/simcore-sdk/tests/integration/np_helpers.py b/packages/simcore-sdk/tests/integration/np_helpers.py index 947f481765ff..343e8e180e81 100644 --- a/packages/simcore-sdk/tests/integration/np_helpers.py +++ b/packages/simcore-sdk/tests/integration/np_helpers.py @@ -3,42 +3,47 @@ # pylint:disable=redefined-outer-name # pylint:disable=too-many-arguments -import json import logging from pathlib import Path +from typing import Dict +import sqlalchemy as sa from simcore_sdk.models.pipeline_models import ComputationalTask log = logging.getLogger(__name__) -def update_configuration(session, project_id, node_uuid, new_configuration): - log.debug("Update configuration of pipeline %s, node %s, on session %s", project_id, node_uuid, session) + +def update_configuration( + postgres_session: sa.orm.session.Session, + project_id: str, + node_uuid: str, + new_configuration: Dict, +) -> None: + log.debug( + "Update configuration of pipeline %s, node %s, on session %s", + project_id, + node_uuid, + postgres_session, + ) # pylint: disable=no-member - task = session.query(ComputationalTask).filter( - ComputationalTask.project_id==str(project_id), - ComputationalTask.node_id==str(node_uuid)) - task.update(dict(schema=new_configuration["schema"], inputs=new_configuration["inputs"], outputs=new_configuration["outputs"])) - session.commit() + task = postgres_session.query(ComputationalTask).filter( + ComputationalTask.project_id == str(project_id), + ComputationalTask.node_id == str(node_uuid), + ) + task.update( + dict( + schema=new_configuration["schema"], + inputs=new_configuration["inputs"], + outputs=new_configuration["outputs"], + ) + ) + postgres_session.commit() log.debug("Updated configuration") -def update_config_file(path, config): - - with open(path, "w") as json_file: - json.dump(config, json_file) - - -def get_empty_config(): - return { - "version": "0.1", - "schema": {"inputs":{}, "outputs":{}}, - "inputs": {}, - "outputs": {} - } - - SIMCORE_STORE = "0" -def file_uuid(file_path:Path, project_id:str, node_uuid:str): - file_id = "{}/{}/{}".format(project_id, node_uuid, Path(file_path).name) + +def file_uuid(file_path: Path, project_id: str, node_uuid: str) -> str: + file_id = f"{project_id}/{node_uuid}/{Path(file_path).name}" return file_id diff --git a/packages/simcore-sdk/tests/integration/test_dbmanager.py b/packages/simcore-sdk/tests/integration/test_dbmanager.py index e3846c10678d..b85d3debda5b 100644 --- a/packages/simcore-sdk/tests/integration/test_dbmanager.py +++ b/packages/simcore-sdk/tests/integration/test_dbmanager.py @@ -2,9 +2,10 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import asyncio import json from pathlib import Path -from typing import Dict +from typing import Callable, Dict from simcore_sdk.node_ports import config from simcore_sdk.node_ports.dbmanager import DBManager @@ -17,7 +18,9 @@ async def test_db_manager_read_config( - loop, nodeports_config, default_configuration: Dict + loop: asyncio.events.AbstractEventLoop, + node_ports_config: None, + default_configuration: Dict, ): db_manager = DBManager() ports_configuration_str = await db_manager.get_ports_configuration_from_node_uuid( @@ -29,7 +32,10 @@ async def test_db_manager_read_config( async def test_db_manager_write_config( - loop, nodeports_config, special_configuration, default_configuration_file: Path + loop: asyncio.events.AbstractEventLoop, + node_ports_config: None, + special_configuration: Callable, + default_configuration_file: Path, ): # create an empty config special_configuration() diff --git a/packages/simcore-sdk/tests/integration/test_filemanager.py b/packages/simcore-sdk/tests/integration/test_filemanager.py index b4b990d54408..91e5b9474908 100644 --- a/packages/simcore-sdk/tests/integration/test_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_filemanager.py @@ -15,21 +15,27 @@ async def test_valid_upload_download( - tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location + tmpdir: Path, + bucket: str, + filemanager_cfg: None, + user_id: str, + file_uuid: str, + s3_simcore_location: str, ): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") assert file_path.exists() file_id = file_uuid(file_path) - store = s3_simcore_location - await filemanager.upload_file( - store_id=store, s3_object=file_id, local_file_path=file_path + store_id, e_tag = await filemanager.upload_file( + store_id=s3_simcore_location, s3_object=file_id, local_file_path=file_path ) + assert store_id == s3_simcore_location + assert e_tag download_folder = Path(tmpdir) / "downloads" download_file_path = await filemanager.download_file_from_s3( - store_id=store, s3_object=file_id, local_folder=download_folder + store_id=s3_simcore_location, s3_object=file_id, local_folder=download_folder ) assert download_file_path.exists() assert download_file_path.name == "test.test" @@ -37,7 +43,12 @@ async def test_valid_upload_download( async def test_invalid_file_path( - tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location + tmpdir: Path, + bucket: str, + filemanager_cfg: None, + user_id: str, + file_uuid: str, + s3_simcore_location: str, ): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") @@ -60,7 +71,11 @@ async def test_invalid_file_path( async def test_invalid_fileid( - tmpdir, bucket, filemanager_cfg, user_id, s3_simcore_location + tmpdir: Path, + bucket: str, + filemanager_cfg: None, + user_id: str, + s3_simcore_location: str, ): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") @@ -88,7 +103,12 @@ async def test_invalid_fileid( async def test_invalid_store( - tmpdir, bucket, filemanager_cfg, user_id, file_uuid, s3_simcore_location + tmpdir: Path, + bucket: str, + filemanager_cfg: None, + user_id: str, + file_uuid: str, + s3_simcore_location: str, ): file_path = Path(tmpdir) / "test.test" file_path.write_text("I am a test file") diff --git a/packages/simcore-sdk/tests/integration/test_nodeports.py b/packages/simcore-sdk/tests/integration/test_nodeports.py index b18e4f1b2c9b..9dc74ca41e73 100644 --- a/packages/simcore-sdk/tests/integration/test_nodeports.py +++ b/packages/simcore-sdk/tests/integration/test_nodeports.py @@ -7,12 +7,15 @@ import filecmp import tempfile from pathlib import Path +from typing import Callable, Dict, Type +import np_helpers # pylint: disable=no-name-in-module import pytest +import sqlalchemy as sa from simcore_sdk import node_ports from simcore_sdk.node_ports import exceptions - -import np_helpers # pylint: disable=no-name-in-module +from simcore_sdk.node_ports._item import ItemConcreteValue +from simcore_sdk.node_ports.nodeports import Nodeports core_services = ["postgres", "storage"] @@ -20,7 +23,7 @@ async def _check_port_valid( - ports, config_dict: dict, port_type: str, key_name: str, key + ports: Nodeports, config_dict: Dict, port_type: str, key_name: str, key: str ): assert (await getattr(ports, port_type))[key].key == key_name # check required values @@ -68,7 +71,7 @@ async def _check_port_valid( assert (await getattr(ports, port_type))[key].value == None -async def _check_ports_valid(ports, config_dict: dict, port_type: str): +async def _check_ports_valid(ports: Nodeports, config_dict: Dict, port_type: str): for key in config_dict["schema"][port_type].keys(): # test using "key" name await _check_port_valid(ports, config_dict, port_type, key, key) @@ -77,19 +80,19 @@ async def _check_ports_valid(ports, config_dict: dict, port_type: str): await _check_port_valid(ports, config_dict, port_type, key, key_index) -async def check_config_valid(ports, config_dict: dict): +async def check_config_valid(ports: Nodeports, config_dict: Dict): await _check_ports_valid(ports, config_dict, "inputs") await _check_ports_valid(ports, config_dict, "outputs") async def test_default_configuration( - loop, default_configuration + default_configuration: Dict, ): # pylint: disable=W0613, W0621 config_dict = default_configuration await check_config_valid(await node_ports.ports(), config_dict) -async def test_invalid_ports(loop, special_configuration): +async def test_invalid_ports(special_configuration: Callable): config_dict, _, _ = special_configuration() PORTS = await node_ports.ports() await check_config_valid(PORTS, config_dict) @@ -120,7 +123,10 @@ async def test_invalid_ports(loop, special_configuration): ], ) async def test_port_value_accessors( - special_configuration, item_type, item_value, item_pytype + special_configuration: Callable, + item_type: str, + item_value: ItemConcreteValue, + item_pytype: Type, ): # pylint: disable=W0613, W0621 item_key = "some key" config_dict, _, _ = special_configuration( @@ -152,14 +158,14 @@ async def test_port_value_accessors( ], ) async def test_port_file_accessors( - special_configuration, - filemanager_cfg, - s3_simcore_location, - bucket, - item_type, - item_value, - item_pytype, - config_value, + special_configuration: Callable, + filemanager_cfg: None, + s3_simcore_location: str, + bucket: str, + item_type: str, + item_value: str, + item_pytype: Type, + config_value: Dict[str, str], ): # pylint: disable=W0613, W0621 config_dict, project_id, node_uuid = special_configuration( inputs=[("in_1", item_type, config_value)], @@ -188,7 +194,10 @@ async def test_port_file_accessors( assert filecmp.cmp(item_value, await (await PORTS.outputs)["out_34"].get()) -async def test_adding_new_ports(special_configuration, postgres_session): +async def test_adding_new_ports( + special_configuration: Callable, + postgres_session: sa.orm.session.Session, +): config_dict, project_id, node_uuid = special_configuration() PORTS = await node_ports.ports() await check_config_valid(PORTS, config_dict) @@ -230,7 +239,10 @@ async def test_adding_new_ports(special_configuration, postgres_session): await check_config_valid(PORTS, config_dict) -async def test_removing_ports(special_configuration, postgres_session): +async def test_removing_ports( + special_configuration: Callable, + postgres_session: sa.orm.session.Session, +): config_dict, project_id, node_uuid = special_configuration( inputs=[("in_14", "integer", 15), ("in_17", "boolean", False)], outputs=[("out_123", "string", "blahblah"), ("out_2", "number", -12.3)], @@ -269,7 +281,11 @@ async def test_removing_ports(special_configuration, postgres_session): ], ) async def test_get_value_from_previous_node( - special_2nodes_configuration, node_link, item_type, item_value, item_pytype + special_2nodes_configuration: Callable, + node_link: Callable, + item_type: str, + item_value: ItemConcreteValue, + item_pytype: Type, ): config_dict, _, _ = special_2nodes_configuration( prev_node_outputs=[("output_123", item_type, item_value)], @@ -292,15 +308,15 @@ async def test_get_value_from_previous_node( ], ) async def test_get_file_from_previous_node( - special_2nodes_configuration, - project_id, - node_uuid, - filemanager_cfg, - node_link, - store_link, - item_type, - item_value, - item_pytype, + special_2nodes_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + node_link: Callable, + store_link: Callable, + item_type: str, + item_value: str, + item_pytype: Type, ): config_dict, _, _ = special_2nodes_configuration( prev_node_outputs=[ @@ -332,17 +348,17 @@ async def test_get_file_from_previous_node( ], ) async def test_get_file_from_previous_node_with_mapping_of_same_key_name( - special_2nodes_configuration, - project_id, - node_uuid, - filemanager_cfg, - node_link, - store_link, - postgres_session, - item_type, - item_value, - item_alias, - item_pytype, + special_2nodes_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + node_link: Callable, + store_link: Callable, + postgres_session: sa.orm.session.Session, + item_type: str, + item_value: str, + item_alias: str, + item_pytype: Type, ): config_dict, _, this_node_uuid = special_2nodes_configuration( prev_node_outputs=[ @@ -378,18 +394,18 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name( ], ) async def test_file_mapping( - special_configuration, - project_id, - node_uuid, - filemanager_cfg, - s3_simcore_location, - bucket, - store_link, - postgres_session, - item_type, - item_value, - item_alias, - item_pytype, + special_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + s3_simcore_location: str, + bucket: str, + store_link: Callable, + postgres_session: sa.orm.session.Session, + item_type: str, + item_value: str, + item_alias: str, + item_pytype: Type, ): config_dict, project_id, node_uuid = special_configuration( inputs=[("in_1", item_type, store_link(item_value, project_id, node_uuid))], diff --git a/packages/simcore-sdk/tests/integration/test_nodeports2.py b/packages/simcore-sdk/tests/integration/test_nodeports2.py new file mode 100644 index 000000000000..a6b1662b2c26 --- /dev/null +++ b/packages/simcore-sdk/tests/integration/test_nodeports2.py @@ -0,0 +1,460 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +# pylint:disable=too-many-arguments +# pylint:disable=pointless-statement + +import filecmp +import tempfile +from pathlib import Path +from typing import Callable, Dict, Type + +import np_helpers # pylint: disable=no-name-in-module +import pytest +import sqlalchemy as sa +from simcore_sdk import node_ports_v2 +from simcore_sdk.node_ports_v2 import exceptions +from simcore_sdk.node_ports_v2.links import ItemConcreteValue +from simcore_sdk.node_ports_v2.nodeports_v2 import Nodeports + +core_services = ["postgres", "storage"] + +ops_services = ["minio"] + + +async def _check_port_valid( + ports: Nodeports, config_dict: Dict, port_type: str, key_name: str, key: str +): + assert (await getattr(ports, port_type))[key].key == key_name + # check required values + assert (await getattr(ports, port_type))[key].label == config_dict["schema"][ + port_type + ][key_name]["label"] + assert (await getattr(ports, port_type))[key].description == config_dict["schema"][ + port_type + ][key_name]["description"] + assert (await getattr(ports, port_type))[key].property_type == config_dict[ + "schema" + ][port_type][key_name]["type"] + assert (await getattr(ports, port_type))[key].display_order == config_dict[ + "schema" + ][port_type][key_name]["displayOrder"] + # check optional values + if "defaultValue" in config_dict["schema"][port_type][key_name]: + assert (await getattr(ports, port_type))[key].default_value == config_dict[ + "schema" + ][port_type][key_name]["defaultValue"] + else: + assert (await getattr(ports, port_type))[key].default_value == None + if "fileToKeyMap" in config_dict["schema"][port_type][key_name]: + assert (await getattr(ports, port_type))[key].file_to_key_map == config_dict[ + "schema" + ][port_type][key_name]["fileToKeyMap"] + else: + assert (await getattr(ports, port_type))[key].file_to_key_map == None + if "widget" in config_dict["schema"][port_type][key_name]: + assert (await getattr(ports, port_type))[key].widget == config_dict["schema"][ + port_type + ][key_name]["widget"] + else: + assert (await getattr(ports, port_type))[key].widget == None + # check payload values + if key_name in config_dict[port_type]: + if isinstance(config_dict[port_type][key_name], dict): + assert (await getattr(ports, port_type))[key].value.dict( + by_alias=True, exclude_unset=True + ) == config_dict[port_type][key_name] + else: + assert (await getattr(ports, port_type))[key].value == config_dict[ + port_type + ][key_name] + elif "defaultValue" in config_dict["schema"][port_type][key_name]: + assert (await getattr(ports, port_type))[key].value == config_dict["schema"][ + port_type + ][key_name]["defaultValue"] + else: + assert (await getattr(ports, port_type))[key].value == None + + +async def _check_ports_valid(ports: Nodeports, config_dict: Dict, port_type: str): + for key in config_dict["schema"][port_type].keys(): + # test using "key" name + await _check_port_valid(ports, config_dict, port_type, key, key) + # test using index + key_index = list(config_dict["schema"][port_type].keys()).index(key) + await _check_port_valid(ports, config_dict, port_type, key, key_index) + + +async def check_config_valid(ports: Nodeports, config_dict: Dict): + await _check_ports_valid(ports, config_dict, "inputs") + await _check_ports_valid(ports, config_dict, "outputs") + + +@pytest.fixture(scope="session") +def e_tag() -> str: + return "123154654684321-1" + + +async def test_default_configuration( + default_configuration: Dict, +): # pylint: disable=W0613, W0621 + config_dict = default_configuration + await check_config_valid(await node_ports_v2.ports(), config_dict) + + +async def test_invalid_ports(special_configuration: Callable): + config_dict, _, _ = special_configuration() + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + + with pytest.raises(exceptions.UnboundPortError): + (await PORTS.inputs)[0] + + with pytest.raises(exceptions.UnboundPortError): + (await PORTS.outputs)[0] + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype", + [ + ("integer", 26, int), + ("integer", 0, int), + ("integer", -52, int), + ("number", -746.4748, float), + ("number", 0.0, float), + ("number", 4566.11235, float), + ("boolean", False, bool), + ("boolean", True, bool), + ("string", "test-string", str), + ("string", "", str), + ], +) +async def test_port_value_accessors( + special_configuration: Callable, + item_type: str, + item_value: ItemConcreteValue, + item_pytype: Type, +): # pylint: disable=W0613, W0621 + item_key = "some_key" + config_dict, _, _ = special_configuration( + inputs=[(item_key, item_type, item_value)], + outputs=[(item_key, item_type, None)], + ) + + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + + assert isinstance(await (await PORTS.inputs)[item_key].get(), item_pytype) + assert await (await PORTS.inputs)[item_key].get() == item_value + assert await (await PORTS.outputs)[item_key].get() is None + + assert isinstance(await PORTS.get(item_key), item_pytype) + assert await PORTS.get(item_key) == item_value + + await (await PORTS.outputs)[item_key].set(item_value) + assert (await PORTS.outputs)[item_key].value == item_value + assert isinstance(await (await PORTS.outputs)[item_key].get(), item_pytype) + assert await (await PORTS.outputs)[item_key].get() == item_value + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype, config_value", + [ + ("data:*/*", __file__, Path, {"store": "0", "path": __file__}), + ("data:text/*", __file__, Path, {"store": "0", "path": __file__}), + ("data:text/py", __file__, Path, {"store": "0", "path": __file__}), + ], +) +async def test_port_file_accessors( + special_configuration: Callable, + filemanager_cfg: None, + s3_simcore_location: str, + bucket: str, + item_type: str, + item_value: str, + item_pytype: Type, + config_value: Dict[str, str], + e_tag: str, +): # pylint: disable=W0613, W0621 + config_dict, project_id, node_uuid = special_configuration( + inputs=[("in_1", item_type, config_value)], + outputs=[("out_34", item_type, None)], + ) + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + assert await (await PORTS.outputs)["out_34"].get() is None # check emptyness + with pytest.raises(exceptions.InvalidDownloadLinkError): + await (await PORTS.inputs)["in_1"].get() + + # this triggers an upload to S3 + configuration change + await (await PORTS.outputs)["out_34"].set(item_value) + # this is the link to S3 storage + received_file_link = (await PORTS.outputs)["out_34"].value.dict( + by_alias=True, exclude_unset=True + ) + assert received_file_link["store"] == s3_simcore_location + assert ( + received_file_link["path"] + == Path(str(project_id), str(node_uuid), Path(item_value).name).as_posix() + ) + # the eTag is created by the S3 server + assert received_file_link["eTag"] + + # this triggers a download from S3 to a location in /tempdir/simcorefiles/item_key + assert isinstance(await (await PORTS.outputs)["out_34"].get(), item_pytype) + assert (await (await PORTS.outputs)["out_34"].get()).exists() + assert str(await (await PORTS.outputs)["out_34"].get()).startswith( + str(Path(tempfile.gettempdir(), "simcorefiles", "out_34")) + ) + filecmp.clear_cache() + assert filecmp.cmp(item_value, await (await PORTS.outputs)["out_34"].get()) + + +async def test_adding_new_ports( + special_configuration: Callable, + postgres_session: sa.orm.session.Session, +): + config_dict, project_id, node_uuid = special_configuration() + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + + # replace the configuration now, add an input + config_dict["schema"]["inputs"].update( + { + "in_15": { + "label": "additional data", + "description": "here some additional data", + "displayOrder": 2, + "type": "integer", + } + } + ) + config_dict["inputs"].update({"in_15": 15}) + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + + # # replace the configuration now, add an output + config_dict["schema"]["outputs"].update( + { + "out_15": { + "label": "output data", + "description": "a cool output", + "displayOrder": 2, + "type": "boolean", + } + } + ) + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + + +async def test_removing_ports( + special_configuration: Callable, + postgres_session: sa.orm.session.Session, +): + config_dict, project_id, node_uuid = special_configuration( + inputs=[("in_14", "integer", 15), ("in_17", "boolean", False)], + outputs=[("out_123", "string", "blahblah"), ("out_2", "number", -12.3)], + ) # pylint: disable=W0612 + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + # let's remove the first input + del config_dict["schema"]["inputs"]["in_14"] + del config_dict["inputs"]["in_14"] + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + # let's do the same for the second output + del config_dict["schema"]["outputs"]["out_2"] + del config_dict["outputs"]["out_2"] + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype", + [ + ("integer", 26, int), + ("integer", 0, int), + ("integer", -52, int), + ("number", -746.4748, float), + ("number", 0.0, float), + ("number", 4566.11235, float), + ("boolean", False, bool), + ("boolean", True, bool), + ("string", "test-string", str), + ("string", "", str), + ], +) +async def test_get_value_from_previous_node( + special_2nodes_configuration: Callable, + node_link: Callable, + item_type: str, + item_value: ItemConcreteValue, + item_pytype: Type, +): + config_dict, _, _ = special_2nodes_configuration( + prev_node_outputs=[("output_123", item_type, item_value)], + inputs=[("in_15", item_type, node_link("output_123"))], + ) + PORTS = await node_ports_v2.ports() + + await check_config_valid(PORTS, config_dict) + input_value = await (await PORTS.inputs)["in_15"].get() + assert isinstance(input_value, item_pytype) + assert await (await PORTS.inputs)["in_15"].get() == item_value + + +@pytest.mark.parametrize( + "item_type, item_value, item_pytype", + [ + ("data:*/*", __file__, Path), + ("data:text/*", __file__, Path), + ("data:text/py", __file__, Path), + ], +) +async def test_get_file_from_previous_node( + special_2nodes_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + node_link: Callable, + store_link: Callable, + item_type: str, + item_value: str, + item_pytype: Type, +): + config_dict, _, _ = special_2nodes_configuration( + prev_node_outputs=[ + ("output_123", item_type, store_link(item_value, project_id, node_uuid)) + ], + inputs=[("in_15", item_type, node_link("output_123"))], + project_id=project_id, + previous_node_id=node_uuid, + ) + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + file_path = await (await PORTS.inputs)["in_15"].get() + assert isinstance(file_path, item_pytype) + assert file_path == Path( + tempfile.gettempdir(), "simcorefiles", "in_15", Path(item_value).name + ) + assert file_path.exists() + filecmp.clear_cache() + assert filecmp.cmp(file_path, item_value) + + +@pytest.mark.parametrize( + "item_type, item_value, item_alias, item_pytype", + [ + ("data:*/*", __file__, Path(__file__).name, Path), + ("data:*/*", __file__, "some funky name.txt", Path), + ("data:text/*", __file__, "some funky name without extension", Path), + ("data:text/py", __file__, "öä$äö2-34 name without extension", Path), + ], +) +async def test_get_file_from_previous_node_with_mapping_of_same_key_name( + special_2nodes_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + node_link: Callable, + store_link: Callable, + postgres_session: sa.orm.session.Session, + item_type: str, + item_value: str, + item_alias: str, + item_pytype: Type, +): + config_dict, _, this_node_uuid = special_2nodes_configuration( + prev_node_outputs=[ + ("in_15", item_type, store_link(item_value, project_id, node_uuid)) + ], + inputs=[("in_15", item_type, node_link("in_15"))], + project_id=project_id, + previous_node_id=node_uuid, + ) + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + # add a filetokeymap + config_dict["schema"]["inputs"]["in_15"]["fileToKeyMap"] = {item_alias: "in_15"} + np_helpers.update_configuration( + postgres_session, project_id, this_node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + file_path = await (await PORTS.inputs)["in_15"].get() + assert isinstance(file_path, item_pytype) + assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_15", item_alias) + assert file_path.exists() + filecmp.clear_cache() + assert filecmp.cmp(file_path, item_value) + + +@pytest.mark.parametrize( + "item_type, item_value, item_alias, item_pytype", + [ + ("data:*/*", __file__, Path(__file__).name, Path), + ("data:*/*", __file__, "some funky name.txt", Path), + ("data:text/*", __file__, "some funky name without extension", Path), + ("data:text/py", __file__, "öä$äö2-34 name without extension", Path), + ], +) +async def test_file_mapping( + special_configuration: Callable, + project_id: str, + node_uuid: str, + filemanager_cfg: None, + s3_simcore_location: str, + bucket: str, + store_link: Callable, + postgres_session: sa.orm.session.Session, + item_type: str, + item_value: str, + item_alias: str, + item_pytype: Type, +): + config_dict, project_id, node_uuid = special_configuration( + inputs=[("in_1", item_type, store_link(item_value, project_id, node_uuid))], + outputs=[("out_1", item_type, None)], + project_id=project_id, + node_id=node_uuid, + ) + PORTS = await node_ports_v2.ports() + await check_config_valid(PORTS, config_dict) + # add a filetokeymap + config_dict["schema"]["inputs"]["in_1"]["fileToKeyMap"] = {item_alias: "in_1"} + config_dict["schema"]["outputs"]["out_1"]["fileToKeyMap"] = {item_alias: "out_1"} + np_helpers.update_configuration( + postgres_session, project_id, node_uuid, config_dict + ) # pylint: disable=E1101 + await check_config_valid(PORTS, config_dict) + file_path = await (await PORTS.inputs)["in_1"].get() + assert isinstance(file_path, item_pytype) + assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_1", item_alias) + + # let's get it a second time to see if replacing works + file_path = await (await PORTS.inputs)["in_1"].get() + assert isinstance(file_path, item_pytype) + assert file_path == Path(tempfile.gettempdir(), "simcorefiles", "in_1", item_alias) + + # now set + invalid_alias = Path("invalid_alias.fjfj") + with pytest.raises(exceptions.PortNotFound): + await PORTS.set_file_by_keymap(invalid_alias) + + await PORTS.set_file_by_keymap(file_path) + file_id = np_helpers.file_uuid(file_path, project_id, node_uuid) + received_file_link = (await PORTS.outputs)["out_1"].value.dict( + by_alias=True, exclude_unset=True + ) + assert received_file_link["store"] == s3_simcore_location + assert received_file_link["path"] == file_id + # received a new eTag + assert received_file_link["eTag"] diff --git a/packages/simcore-sdk/tests/mock/default_config.json b/packages/simcore-sdk/tests/mock/default_config.json new file mode 100644 index 000000000000..35bdfd9cfc4e --- /dev/null +++ b/packages/simcore-sdk/tests/mock/default_config.json @@ -0,0 +1,59 @@ +{ + "schema": { + "inputs": { + "in_1": { + "displayOrder": 0, + "label": "computational data", + "description": "these are computed data out of a pipeline", + "type": "data:*/*", + "defaultValue": null, + "fileToKeyMap": { + "input1.txt": "in_1" + }, + "widget": null + }, + "in_5": { + "displayOrder": 2, + "label": "some number", + "description": "numbering things", + "type": "integer", + "defaultValue": 666, + "fileToKeyMap": {}, + "widget": null + } + }, + "outputs": { + "out_1": { + "displayOrder": 0, + "label": "some boolean output", + "description": "could be true or false...", + "type": "boolean", + "defaultValue": null, + "fileToKeyMap": {}, + "widget": null + }, + "out_2": { + "displayOrder": 1, + "label": "some file output", + "description": "could be anything...", + "type": "data:*/*", + "defaultValue": null, + "fileToKeyMap": {}, + "widget": null + } + } + }, + "inputs": { + "in_1": { + "nodeUuid": "793bc431-b935-4f9f-8b04-e4117640e9b6", + "output": "outFile" + } + }, + "outputs": { + "out_1": false, + "out_2": { + "store": "z43-s3", + "path": "/simcore/outputControllerOut.dat" + } + } +} diff --git a/packages/simcore-sdk/tests/mock/empty_config.json b/packages/simcore-sdk/tests/mock/empty_config.json new file mode 100644 index 000000000000..33f26d13e6a1 --- /dev/null +++ b/packages/simcore-sdk/tests/mock/empty_config.json @@ -0,0 +1,8 @@ +{ + "schema": { + "inputs": {}, + "outputs": {} + }, + "inputs": {}, + "outputs": {} +} diff --git a/packages/simcore-sdk/tests/unit/conftest.py b/packages/simcore-sdk/tests/unit/conftest.py new file mode 100644 index 000000000000..b1c60061e57b --- /dev/null +++ b/packages/simcore-sdk/tests/unit/conftest.py @@ -0,0 +1,49 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import asyncio +import json +from typing import Any, Callable, Dict +from uuid import uuid4 + +import pytest +from simcore_sdk.node_ports.dbmanager import DBManager + + +@pytest.fixture(scope="module") +def node_uuid() -> str: + return str(uuid4()) + + +@pytest.fixture(scope="function") +async def mock_db_manager( + loop: asyncio.AbstractEventLoop, + monkeypatch, + node_uuid: str, +) -> Callable: + def _mock_db_manager(port_cfg: Dict[str, Any]) -> DBManager: + async def mock_get_ports_configuration_from_node_uuid(*args, **kwargs) -> str: + return json.dumps(port_cfg) + + async def mock_write_ports_configuration( + self, json_configuration: str, uuid: str + ): + assert json.loads(json_configuration) == port_cfg + assert uuid == node_uuid + + monkeypatch.setattr( + DBManager, + "get_ports_configuration_from_node_uuid", + mock_get_ports_configuration_from_node_uuid, + ) + monkeypatch.setattr( + DBManager, + "write_ports_configuration", + mock_write_ports_configuration, + ) + + db_manager = DBManager() + return db_manager + + yield _mock_db_manager diff --git a/packages/simcore-sdk/tests/unit/test_data_item.py b/packages/simcore-sdk/tests/unit/test_data_item.py index e4b382d61334..a17e5f35d70c 100644 --- a/packages/simcore-sdk/tests/unit/test_data_item.py +++ b/packages/simcore-sdk/tests/unit/test_data_item.py @@ -4,7 +4,7 @@ import pytest from simcore_sdk.node_ports import exceptions -from simcore_sdk.node_ports._data_item import DataItem +from simcore_sdk.node_ports._data_item import DataItem, DataItemValue @pytest.mark.parametrize( @@ -37,7 +37,7 @@ (None), ], ) -def test_default_item(item_value): +def test_default_item(item_value: DataItemValue): with pytest.raises(exceptions.InvalidProtocolError): DataItem() with pytest.raises(exceptions.InvalidProtocolError): diff --git a/packages/simcore-sdk/tests/unit/test_data_manager.py b/packages/simcore-sdk/tests/unit/test_data_manager.py index 19a91db929e7..d69d3d3fbc64 100644 --- a/packages/simcore-sdk/tests/unit/test_data_manager.py +++ b/packages/simcore-sdk/tests/unit/test_data_manager.py @@ -2,20 +2,22 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import asyncio from asyncio import Future from filecmp import cmpfiles from pathlib import Path -from shutil import make_archive, unpack_archive, copy +from shutil import copy, make_archive, unpack_archive +from typing import Callable, List import pytest from simcore_sdk.node_data import data_manager @pytest.fixture -def create_files(): +def create_files() -> Callable: created_files = [] - def _create_files(number: int, folder: Path): + def _create_files(number: int, folder: Path) -> List[Path]: for i in range(number): file_path = folder / "{}.test".format(i) @@ -30,7 +32,9 @@ def _create_files(number: int, folder: Path): file_path.unlink() -async def test_push_folder(loop, mocker, tmpdir, create_files): +async def test_push_folder( + loop: asyncio.events.AbstractEventLoop, mocker, tmpdir: Path, create_files: Callable +): # create some files assert tmpdir.exists() @@ -93,7 +97,9 @@ async def test_push_folder(loop, mocker, tmpdir, create_files): assert not errors -async def test_push_file(loop, mocker, tmpdir, create_files): +async def test_push_file( + loop: asyncio.events.AbstractEventLoop, mocker, tmpdir: Path, create_files: Callable +): mock_filemanager = mocker.patch( "simcore_sdk.node_data.data_manager.filemanager", spec=True ) @@ -122,7 +128,9 @@ async def test_push_file(loop, mocker, tmpdir, create_files): mock_filemanager.reset_mock() -async def test_pull_folder(loop, mocker, tmpdir, create_files): +async def test_pull_folder( + loop: asyncio.events.AbstractEventLoop, mocker, tmpdir: Path, create_files: Callable +): assert tmpdir.exists() # create a folder to compress from test_control_folder = Path(tmpdir) / "test_control_folder" @@ -186,7 +194,9 @@ async def test_pull_folder(loop, mocker, tmpdir, create_files): assert not errors -async def test_pull_file(loop, mocker, tmpdir, create_files): +async def test_pull_file( + loop: asyncio.events.AbstractEventLoop, mocker, tmpdir: Path, create_files: Callable +): file_path = create_files(1, Path(tmpdir))[0] assert file_path.exists() assert file_path.is_file() diff --git a/packages/simcore-sdk/tests/unit/test_item.py b/packages/simcore-sdk/tests/unit/test_item.py index 0f12a234e044..3d802c9efec8 100644 --- a/packages/simcore-sdk/tests/unit/test_item.py +++ b/packages/simcore-sdk/tests/unit/test_item.py @@ -4,12 +4,12 @@ # pylint:disable=no-member from pathlib import Path -from typing import Callable, Dict, Union +from typing import Callable import pytest from simcore_sdk.node_ports import config, data_items_utils, exceptions, filemanager from simcore_sdk.node_ports._data_item import DataItem -from simcore_sdk.node_ports._item import Item +from simcore_sdk.node_ports._item import DataItemValue, Item, ItemConcreteValue from simcore_sdk.node_ports._schema_item import SchemaItem from utils_futures import future_with_result @@ -19,7 +19,7 @@ def node_ports_config(): config.STORAGE_ENDPOINT = "storage:8080" -def create_item(item_type: str, item_value): +def create_item(item_type: str, item_value: DataItemValue) -> Item: key = "some key" return Item( SchemaItem( @@ -78,7 +78,7 @@ async def test_valid_type_empty_value(item_type: str): @pytest.fixture async def file_link_mock( - monkeypatch, item_type: str, item_value: Union[int, float, bool, str, Dict] + monkeypatch, item_type: str, item_value: DataItemValue ) -> Callable: async def fake_download_file(*args, **kwargs) -> Path: return_value = "mydefault" @@ -128,7 +128,7 @@ async def test_valid_type( loop, file_link_mock: Callable, item_type: str, - item_value: Union[int, float, bool, str, Dict], + item_value: DataItemValue, ): item = create_item(item_type, item_value) if not data_items_utils.is_value_link(item_value): @@ -203,7 +203,7 @@ async def test_valid_type( ), ], ) -async def test_invalid_type(item_type, item_value): +async def test_invalid_type(item_type: str, item_value: DataItemValue): # pylint: disable=W0612 with pytest.raises( exceptions.InvalidItemTypeError, match=rf"Invalid item type, .*[{item_type}]" @@ -216,13 +216,15 @@ async def test_invalid_type(item_type, item_value): [ ("integer", 26, 26), ("number", -746.4748, -746.4748), - # ("data:*/*", __file__, {"store":"s3-z43", "path":"undefined/undefined/{filename}".format(filename=Path(__file__).name)}), ("boolean", False, False), ("string", "test-string", "test-string"), ], ) async def test_set_new_value( - item_type, item_value_to_set, expected_value, mocker + item_type: str, + item_value_to_set: ItemConcreteValue, + expected_value: ItemConcreteValue, + mocker, ): # pylint: disable=W0613 mock_method = mocker.Mock(return_value=future_with_result("")) item = create_item(item_type, None) @@ -237,13 +239,14 @@ async def test_set_new_value( [ ("integer", -746.4748), ("number", "a string"), + ("data:*/*", Path(__file__).parent), ("data:*/*", str(Path(__file__).parent)), ("boolean", 123), ("string", True), ], ) async def test_set_new_invalid_value( - item_type, item_value_to_set + item_type: str, item_value_to_set: ItemConcreteValue ): # pylint: disable=W0613 item = create_item(item_type, None) assert await item.get() is None diff --git a/packages/simcore-sdk/tests/unit/test_itemstlist.py b/packages/simcore-sdk/tests/unit/test_itemstlist.py index b1244adeb62a..7c4723c1b4b6 100644 --- a/packages/simcore-sdk/tests/unit/test_itemstlist.py +++ b/packages/simcore-sdk/tests/unit/test_itemstlist.py @@ -3,8 +3,9 @@ # pylint:disable=redefined-outer-name # pylint:disable=no-member -import pytest +from typing import Dict, Union +import pytest from simcore_sdk.node_ports._data_item import DataItem from simcore_sdk.node_ports._data_items_list import DataItemsList from simcore_sdk.node_ports._item import Item @@ -14,7 +15,9 @@ from utils_futures import future_with_result -def create_item(key, item_type, item_value): +def create_item( + key: str, item_type: str, item_value: Union[int, float, bool, str, Dict] +): return Item( SchemaItem( key=key, diff --git a/packages/simcore-sdk/tests/unit/test_links.py b/packages/simcore-sdk/tests/unit/test_links.py new file mode 100644 index 000000000000..5116311ae01a --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_links.py @@ -0,0 +1,54 @@ +from typing import Dict +from uuid import uuid4 + +import pytest +from pydantic import ValidationError +from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink + + +def test_valid_port_link(): + port_link = {"nodeUuid": f"{uuid4()}", "output": "some_key"} + PortLink(**port_link) + + +@pytest.mark.parametrize( + "port_link", + [ + {"nodeUuid": f"{uuid4()}"}, + {"output": "some_stuff"}, + {"nodeUuid": "some stuff", "output": "some_stuff"}, + {"nodeUuid": "", "output": "some stuff"}, + {"nodeUuid": f"{uuid4()}", "output": ""}, + {"nodeUuid": f"{uuid4()}", "output": "some.key"}, + {"nodeUuid": f"{uuid4()}", "output": "some:key"}, + ], +) +def test_invalid_port_link(port_link: Dict[str, str]): + with pytest.raises(ValidationError): + PortLink(**port_link) + + +@pytest.mark.parametrize( + "download_link", + [ + {"downloadLink": ""}, + {"downloadLink": "some stuff"}, + {"label": "some stuff"}, + ], +) +def test_invalid_download_link(download_link: Dict[str, str]): + with pytest.raises(ValidationError): + DownloadLink(**download_link) + + +@pytest.mark.parametrize( + "file_link", + [ + {"store": ""}, + {"store": "0", "path": ""}, + {"path": "/somefile/blahblah:"}, + ], +) +def test_invalid_file_link(file_link: Dict[str, str]): + with pytest.raises(ValidationError): + FileLink(**file_link) diff --git a/packages/simcore-sdk/tests/unit/test_nodeports_v2.py b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py new file mode 100644 index 000000000000..841b4b22ce7e --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_nodeports_v2.py @@ -0,0 +1,182 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +from asyncio import Future +from pathlib import Path +from typing import Any, Callable, Dict + +import pytest +from simcore_sdk.node_ports_v2 import Nodeports, exceptions, ports +from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList +from utils_port_v2 import create_valid_port_mapping + + +@pytest.mark.parametrize( + "auto_update", + [ + pytest.param(True, id="Autoupdate enabled"), + pytest.param(False, id="Autoupdate disabled"), + ], +) +async def test_nodeports_auto_updates( + mock_db_manager: Callable, + default_configuration: Dict[str, Any], + node_uuid: str, + auto_update: bool, +): + db_manager = mock_db_manager(default_configuration) + + original_inputs = create_valid_port_mapping(InputsList, suffix="original") + original_outputs = create_valid_port_mapping(OutputsList, suffix="original") + + updated_inputs = create_valid_port_mapping(InputsList, suffix="updated") + updated_outputs = create_valid_port_mapping(OutputsList, suffix="updated") + + async def mock_save_db_cb(*args, **kwargs): + pass + + async def mock_node_port_creator_cb(*args, **kwargs): + updated_node_ports = Nodeports( + inputs=updated_inputs, + outputs=updated_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + return updated_node_ports + + node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=auto_update, + ) + + assert node_ports.internal_inputs == original_inputs + assert node_ports.internal_outputs == original_outputs + + # this triggers an auto_update if auto_update is True + node_inputs = await node_ports.inputs + assert node_inputs == updated_inputs if auto_update else original_inputs + node_outputs = await node_ports.outputs + assert node_outputs == updated_outputs if auto_update else original_outputs + + +async def test_node_ports_accessors( + mock_db_manager: Callable, + default_configuration: Dict[str, Any], + node_uuid: str, +): + db_manager = mock_db_manager(default_configuration) + + original_inputs = create_valid_port_mapping(InputsList, suffix="original") + original_outputs = create_valid_port_mapping(OutputsList, suffix="original") + + async def mock_save_db_cb(*args, **kwargs): + pass + + async def mock_node_port_creator_cb(*args, **kwargs): + updated_node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + return updated_node_ports + + node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + + for port in original_inputs.values(): + assert await node_ports.get(port.key) == port.value + await node_ports.set(port.key, port.value) + + with pytest.raises(exceptions.UnboundPortError): + await node_ports.get("some_invalid_key") + + for port in original_outputs.values(): + assert await node_ports.get(port.key) == port.value + await node_ports.set(port.key, port.value) + + +@pytest.fixture(scope="session") +def e_tag() -> str: + return "123154654684321-1" + + +@pytest.fixture +async def mock_upload_file(mocker, e_tag): + mock = mocker.patch( + "simcore_sdk.node_ports.filemanager.upload_file", + return_value=Future(), + ) + mock.return_value.set_result(("0", e_tag)) + yield mock + + +async def test_node_ports_set_file_by_keymap( + mock_db_manager: Callable, + default_configuration: Dict[str, Any], + node_uuid: str, + mock_upload_file, +): + db_manager = mock_db_manager(default_configuration) + + original_inputs = create_valid_port_mapping(InputsList, suffix="original") + original_outputs = create_valid_port_mapping( + OutputsList, suffix="original", file_to_key=Path(__file__).name + ) + + async def mock_save_db_cb(*args, **kwargs): + pass + + async def mock_node_port_creator_cb(*args, **kwargs): + updated_node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + return updated_node_ports + + node_ports = Nodeports( + inputs=original_inputs, + outputs=original_outputs, + db_manager=db_manager, + node_uuid=node_uuid, + save_to_db_cb=mock_save_db_cb, + node_port_creator_cb=mock_node_port_creator_cb, + auto_update=False, + ) + + await node_ports.set_file_by_keymap(Path(__file__)) + + with pytest.raises(exceptions.PortNotFound): + await node_ports.set_file_by_keymap(Path("/whatever/file/that/is/invalid")) + + +async def test_node_ports_v2_packages( + mock_db_manager: Callable, default_configuration: Dict[str, Any] +): + db_manager = mock_db_manager(default_configuration) + node_ports = await ports() + node_ports = await ports(db_manager) diff --git a/packages/simcore-sdk/tests/unit/test_package.py b/packages/simcore-sdk/tests/unit/test_package.py new file mode 100644 index 000000000000..5b5a4338d3b9 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_package.py @@ -0,0 +1,38 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +import os +import re +from pathlib import Path + +import pytest +from pytest_simcore.helpers.utils_pylint import assert_pylint_is_passing + + +@pytest.fixture +def pylintrc(osparc_simcore_root_dir): + pylintrc = osparc_simcore_root_dir / ".pylintrc" + assert pylintrc.exists() + return pylintrc + + +def test_run_pylint(pylintrc, package_dir): + assert_pylint_is_passing(pylintrc=pylintrc, package_dir=package_dir) + + +def test_no_pdbs_in_place(package_dir): + # TODO: add also test_dir excluding this function!? + # TODO: it can be commented! + # TODO: add check on other undesired code strings?! + MATCH = re.compile(r"pdb.set_trace()") + EXCLUDE = ["__pycache__", ".git"] + for root, dirs, files in os.walk(package_dir): + for name in files: + if name.endswith(".py"): + pypth = Path(root) / name + code = pypth.read_text() + found = MATCH.findall(code) + # TODO: should return line number + assert not found, "pbd.set_trace found in %s" % pypth + dirs[:] = [d for d in dirs if d not in EXCLUDE] diff --git a/packages/simcore-sdk/tests/unit/test_port.py b/packages/simcore-sdk/tests/unit/test_port.py new file mode 100644 index 000000000000..b688e19775e9 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_port.py @@ -0,0 +1,652 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +# pylint:disable=no-member +# pylint:disable=protected-access +# pylint:disable=too-many-arguments +import re +import shutil +import tempfile +from asyncio import Future +from collections import namedtuple +from pathlib import Path +from typing import Any, Dict, Optional, Type, Union + +import pytest +from aiohttp.client import ClientSession +from pydantic.error_wrappers import ValidationError +from simcore_sdk.node_ports_v2 import exceptions, node_config +from simcore_sdk.node_ports_v2.links import DownloadLink, FileLink, PortLink +from simcore_sdk.node_ports_v2.port import Port +from utils_port_v2 import create_valid_port_config +from yarl import URL + + +##################### HELPERS +def camel_to_snake(name): + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() + + +PortParams = namedtuple( + "PortParams", + "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", +) + + +def this_node_file_name() -> Path: + return Path(tempfile.gettempdir(), "this_node_file.txt") + + +def another_node_file_name() -> Path: + return Path(tempfile.gettempdir(), "another_node_file.txt") + + +def download_file_folder_name() -> Path: + return Path(tempfile.gettempdir(), "simcorefiles") + + +def project_id() -> str: + return "cd0d8dbb-3263-44dc-921c-49c075ac0dd9" + + +def node_uuid() -> str: + return "609b7af4-6861-4aa7-a16e-730ea8125190" + + +def user_id() -> str: + return "666" + + +def simcore_store_id() -> str: + return "0" + + +def datcore_store_id() -> str: + return "1" + + +def e_tag() -> str: + return "1212132546546321-1" + + +##################### FIXTURES + + +@pytest.fixture +def this_node_file(tmp_path: Path) -> Path: + file_path = this_node_file_name() + file_path.write_text("some dummy data") + assert file_path.exists() + yield file_path + if file_path.exists(): + file_path.unlink() + + +@pytest.fixture +def another_node_file() -> Path: + file_path = another_node_file_name() + file_path.write_text("some dummy data") + assert file_path.exists() + yield file_path + if file_path.exists(): + file_path.unlink() + + +@pytest.fixture +def download_file_folder() -> Path: + destination_path = download_file_folder_name() + destination_path.mkdir(parents=True, exist_ok=True) + yield destination_path + if destination_path.exists(): + shutil.rmtree(destination_path) + + +@pytest.fixture(scope="module", name="project_id") +def project_id_fixture() -> str: + """NOTE: since pytest does not allow to use fixtures inside parametrizations, + this trick allows to re-use the same function in a fixture with a same "fixture" name""" + return project_id() + + +@pytest.fixture(scope="module", name="node_uuid") +def node_uuid_fixture() -> str: + """NOTE: since pytest does not allow to use fixtures inside parametrizations, + this trick allows to re-use the same function in a fixture with a same "fixture" name""" + return node_uuid() + + +@pytest.fixture(scope="module", name="user_id") +def user_id_fixture() -> str: + """NOTE: since pytest does not allow to use fixtures inside parametrizations, + this trick allows to re-use the same function in a fixture with a same "fixture" name""" + return user_id() + + +@pytest.fixture +async def mock_download_file( + monkeypatch, + this_node_file: Path, + project_id: str, + node_uuid: str, + download_file_folder: Path, +): + async def mock_download_file_from_link( + download_link: URL, + local_folder: Path, + session: Optional[ClientSession] = None, + file_name: Optional[str] = None, + ) -> Path: + assert str(local_folder).startswith(str(download_file_folder)) + destination_path = local_folder / this_node_file.name + destination_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(this_node_file, destination_path) + return destination_path + + from simcore_sdk.node_ports import filemanager + + monkeypatch.setattr( + filemanager, "download_file_from_link", mock_download_file_from_link + ) + + +@pytest.fixture(scope="session", name="e_tag") +def e_tag_fixture() -> str: + return "1212132546546321-1" + + +@pytest.fixture +async def mock_upload_file(mocker, e_tag): + mock = mocker.patch( + "simcore_sdk.node_ports.filemanager.upload_file", + return_value=Future(), + ) + mock.return_value.set_result((simcore_store_id(), e_tag)) + yield mock + + +@pytest.fixture(autouse=True) +def common_fixtures( + loop, + storage_v0_service_mock, + mock_download_file, + mock_upload_file, + project_id: str, + user_id: str, + node_uuid: str, + this_node_file: Path, + another_node_file: Path, + download_file_folder: Path, +): + """this module main fixture""" + + node_config.USER_ID = user_id + node_config.PROJECT_ID = project_id + node_config.NODE_UUID = node_uuid + node_config.STORAGE_ENDPOINT = "storage:8080" + + +##################### TESTS +@pytest.mark.parametrize( + "port_cfg, exp_value_type, exp_value_converter, exp_value, exp_get_value, new_value, exp_new_value, exp_new_get_value", + [ + pytest.param( + *PortParams( + port_cfg=create_valid_port_config("integer", defaultValue=3), + exp_value_type=(int), + exp_value_converter=int, + exp_value=3, + exp_get_value=3, + new_value=7, + exp_new_value=7, + exp_new_get_value=7, + ), + id="integer value with default", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config("number", defaultValue=-23.45), + exp_value_type=(float), + exp_value_converter=float, + exp_value=-23.45, + exp_get_value=-23.45, + new_value=7, + exp_new_value=7.0, + exp_new_get_value=7.0, + ), + id="number value with default", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config("boolean", defaultValue=True), + exp_value_type=(bool), + exp_value_converter=bool, + exp_value=True, + exp_get_value=True, + new_value=False, + exp_new_value=False, + exp_new_get_value=False, + ), + id="boolean value with default", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "boolean", defaultValue=True, value=False + ), + exp_value_type=(bool), + exp_value_converter=bool, + exp_value=False, + exp_get_value=False, + new_value=True, + exp_new_value=True, + exp_new_get_value=True, + ), + id="boolean value with default and value", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config("data:*/*", key="no_file"), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=None, + exp_get_value=None, + new_value=str(this_node_file_name()), + exp_new_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), + ), + exp_new_get_value=download_file_folder_name() + / "no_file" + / this_node_file_name().name, + ), + id="file type with no payload", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="no_file_with_default", + defaultValue=str(this_node_file_name()), + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=None, + exp_get_value=None, + new_value=this_node_file_name(), + exp_new_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), + ), + exp_new_get_value=download_file_folder_name() + / "no_file_with_default" + / this_node_file_name().name, + ), + id="file link with no payload and default value", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="some_file", + value={ + "store": simcore_store_id(), + "path": f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + ), + exp_get_value=download_file_folder_name() + / "some_file" + / this_node_file_name().name, + new_value=None, + exp_new_value=None, + exp_new_get_value=None, + ), + id="file link with payload that gets reset", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="some_file_with_file_to_key_map", + fileToKeyMap={ + "a_new_fancy_name.csv": "some_file_with_file_to_key_map" + }, + value={ + "store": simcore_store_id(), + "path": f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + ), + exp_get_value=download_file_folder_name() + / "some_file_with_file_to_key_map" + / "a_new_fancy_name.csv", + new_value=None, + exp_new_value=None, + exp_new_get_value=None, + ), + id="file link with fileToKeyMap with payload that gets reset", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="some_file_on_datcore", + value={ + "store": datcore_store_id(), + "path": f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + "dataset": "some blahblah", + "label": "some blahblah", + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=FileLink( + store=datcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + dataset="some blahblah", + label="some blahblah", + ), + exp_get_value=download_file_folder_name() + / "some_file_on_datcore" + / this_node_file_name().name, + new_value=this_node_file_name(), + exp_new_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), + ), + exp_new_get_value=download_file_folder_name() + / "some_file_on_datcore" + / this_node_file_name().name, + ), + id="file link with payload on store 1", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="download_link", + value={ + "downloadLink": "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=DownloadLink( + downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" + ), + exp_get_value=download_file_folder_name() + / "download_link" + / this_node_file_name().name, + new_value=this_node_file_name(), + exp_new_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), + ), + exp_new_get_value=download_file_folder_name() + / "download_link" + / this_node_file_name().name, + ), + id="download link file type gets set back on store", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="download_link_with_file_to_key", + fileToKeyMap={ + "a_cool_file_type.zip": "download_link_with_file_to_key" + }, + value={ + "downloadLink": "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=DownloadLink( + downloadLink="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/master/README.md" + ), + exp_get_value=download_file_folder_name() + / "download_link_with_file_to_key" + / "a_cool_file_type.zip", + new_value=this_node_file_name(), + exp_new_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), + ), + exp_new_get_value=download_file_folder_name() + / "download_link_with_file_to_key" + / "a_cool_file_type.zip", + ), + id="download link file type with filetokeymap gets set back on store", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="file_port_link", + value={ + "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + "output": "the_output_of_that_node", + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=PortLink( + nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + output="the_output_of_that_node", + ), + exp_get_value=download_file_folder_name() + / "file_port_link" + / another_node_file_name().name, + new_value=this_node_file_name(), + exp_new_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), + ), + exp_new_get_value=download_file_folder_name() + / "file_port_link" + / this_node_file_name().name, + ), + id="file node link type gets set back on store", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "data:*/*", + key="file_port_link_with_file_to_key_map", + fileToKeyMap={ + "a_cool_file_type.zip": "file_port_link_with_file_to_key_map" + }, + value={ + "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + "output": "the_output_of_that_node", + }, + ), + exp_value_type=(Path, str), + exp_value_converter=Path, + exp_value=PortLink( + nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + output="the_output_of_that_node", + ), + exp_get_value=download_file_folder_name() + / "file_port_link_with_file_to_key_map" + / "a_cool_file_type.zip", + new_value=this_node_file_name(), + exp_new_value=FileLink( + store=simcore_store_id(), + path=f"{project_id()}/{node_uuid()}/{this_node_file_name().name}", + e_tag=e_tag(), + ), + exp_new_get_value=download_file_folder_name() + / "file_port_link_with_file_to_key_map" + / "a_cool_file_type.zip", + ), + id="file node link type with file to key map gets set back on store", + ), + pytest.param( + *PortParams( + port_cfg=create_valid_port_config( + "number", + key="number_port_link", + value={ + "nodeUuid": "238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + "output": "the_output_of_that_node", + }, + ), + exp_value_type=(float), + exp_value_converter=float, + exp_value=PortLink( + nodeUuid="238e5b86-ed65-44b0-9aa4-f0e23ca8a083", + output="the_output_of_that_node", + ), + exp_get_value=562.45, + new_value=None, + exp_new_value=None, + exp_new_get_value=None, + ), + id="number node link type gets reset", + ), + ], +) +async def test_valid_port( + port_cfg: Dict[str, Any], + exp_value_type: Type[Union[int, float, bool, str, Path]], + exp_value_converter: Type[Union[int, float, bool, str, Path]], + exp_value: Union[int, float, bool, str, Path, FileLink, DownloadLink, PortLink], + exp_get_value: Union[int, float, bool, str, Path], + new_value: Union[int, float, bool, str, Path], + exp_new_value: Union[int, float, bool, str, Path, FileLink], + exp_new_get_value: Union[int, float, bool, str, Path], + another_node_file: Path, +): + class FakeNodePorts: + async def get(self, key): + # this gets called when a node links to another node we return the get value but for files it needs to be a real one + return ( + another_node_file + if port_cfg["type"].startswith("data:") + else exp_get_value + ) + + async def _node_ports_creator_cb(self, node_uuid: str): + return FakeNodePorts() + + async def save_to_db_cb(self, node_ports): + return + + fake_node_ports = FakeNodePorts() + port = Port(**port_cfg) + port._node_ports = fake_node_ports + + # check schema + for k, v in port_cfg.items(): + camel_key = camel_to_snake(k) + if k == "type": + camel_key = "property_type" + if k != "value": + assert v == getattr(port, camel_key) + + # check payload + assert port._py_value_type == exp_value_type + assert port._py_value_converter == exp_value_converter + + assert port.value == exp_value + + if isinstance(exp_get_value, Path): + # if it's a file let's create one there already + exp_get_value.parent.mkdir(parents=True, exist_ok=True) + exp_get_value.touch() + + if exp_get_value is None: + assert await port.get() == None + else: + assert await port.get() == exp_get_value + if isinstance(exp_value, PortLink) and isinstance(exp_get_value, Path): + # as the file is moved internally we need to re-create it or it fails + another_node_file_name().touch(exist_ok=True) + # it should work several times + assert await port.get() == exp_get_value + + # set a new value + await port.set(new_value) + assert port.value == exp_new_value + + if isinstance(exp_new_get_value, Path): + # if it's a file let's create one there already + exp_new_get_value.parent.mkdir(parents=True, exist_ok=True) + exp_new_get_value.touch() + if exp_new_get_value is None: + assert await port.get() == None + else: + assert await port.get() == exp_new_get_value + assert await port.get() == exp_new_get_value + + +@pytest.mark.parametrize( + "port_cfg", + [ + { + "key": "some.key", + "label": "some label", + "description": "some description", + "type": "integer", + "displayOrder": 2.3, + }, + { + "key": "some:key", + "label": "", + "description": "", + "type": "integer", + "displayOrder": 2.3, + }, + { + "key": "some_key", + "label": "", + "description": "", + "type": "blahblah", + "displayOrder": 2.3, + }, + { + "key": "some_file_with_file_in_value", + "label": "", + "description": "", + "type": "data:*/*", + "displayOrder": 2.3, + "value": __file__, + }, + ], +) +def test_invalid_port(port_cfg: Dict[str, Any]): + with pytest.raises(ValidationError): + Port(**port_cfg) + + +@pytest.mark.parametrize( + "port_cfg", [(create_valid_port_config("data:*/*", key="set_some_inexisting_file"))] +) +async def test_invalid_file_type_setter(port_cfg: Dict[str, Any]): + port = Port(**port_cfg) + # set a file that does not exist + with pytest.raises(exceptions.InvalidItemTypeError): + await port.set("some/dummy/file/name") + + # set a folder fails too + with pytest.raises(exceptions.InvalidItemTypeError): + await port.set(Path(__file__).parent) diff --git a/packages/simcore-sdk/tests/unit/test_port_mapping.py b/packages/simcore-sdk/tests/unit/test_port_mapping.py new file mode 100644 index 000000000000..e1395c376366 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_port_mapping.py @@ -0,0 +1,46 @@ +from typing import Any, Dict, Type, Union + +import pytest +from simcore_sdk.node_ports_v2 import exceptions +from simcore_sdk.node_ports_v2.ports_mapping import InputsList, OutputsList +from utils_port_v2 import create_valid_port_config + + +##################### TESTS +@pytest.mark.parametrize("port_class", [InputsList, OutputsList]) +def test_empty_ports_mapping(port_class: Type[Union[InputsList, OutputsList]]): + port_mapping = port_class(**{"__root__": {}}) + assert not port_mapping.items() + assert not port_mapping.values() + assert not port_mapping.keys() + assert len(port_mapping) == 0 + for port_key in port_mapping: + # it should be empty + assert True, f"should be empty, got {port_key}" + + +@pytest.mark.parametrize("port_class", [InputsList, OutputsList]) +def test_filled_ports_mapping(port_class: Type[Union[InputsList, OutputsList]]): + port_cfgs: Dict[str, Any] = {} + for t in ["integer", "number", "boolean", "string"]: + port = create_valid_port_config(t) + port_cfgs[port["key"]] = port + port_cfgs["some_file"] = create_valid_port_config("data:*/*", key="some_file") + port_mapping = port_class(**{"__root__": port_cfgs}) + + assert len(port_mapping) == len(port_cfgs) + for port_key, port_value in port_mapping.items(): + assert port_key in port_mapping + assert port_key in port_cfgs + + # just to make use of the variable and check the pydantic overloads are working correctly + assert port_mapping[port_key] == port_value + + for index, port_key in enumerate(port_cfgs): + assert port_mapping[index] == port_mapping[port_key] + + with pytest.raises(exceptions.UnboundPortError): + _ = port_mapping[len(port_cfgs)] + + with pytest.raises(exceptions.UnboundPortError): + _ = port_mapping["whatever"] diff --git a/packages/simcore-sdk/tests/unit/test_schema_item.py b/packages/simcore-sdk/tests/unit/test_schema_item.py index 24bc48dd6e31..0701670d209a 100644 --- a/packages/simcore-sdk/tests/unit/test_schema_item.py +++ b/packages/simcore-sdk/tests/unit/test_schema_item.py @@ -2,17 +2,24 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -import pytest from copy import deepcopy -from simcore_sdk.node_ports import exceptions, config + +import pytest +from simcore_sdk.node_ports import config, exceptions from simcore_sdk.node_ports._schema_item import SchemaItem + def test_default_item(): with pytest.raises(exceptions.InvalidProtocolError): - item = SchemaItem() #pylint: disable=W0612 + item = SchemaItem() # pylint: disable=W0612 -def test_check_item_required_fields(): #pylint: disable=W0612 - required_parameters = {key:"defaultValue" for key, required in config.SCHEMA_ITEM_KEYS.items() if required} + +def test_check_item_required_fields(): # pylint: disable=W0612 + required_parameters = { + key: "defaultValue" + for key, required in config.SCHEMA_ITEM_KEYS.items() + if required + } # this shall not trigger an exception SchemaItem(**required_parameters) @@ -22,8 +29,15 @@ def test_check_item_required_fields(): #pylint: disable=W0612 with pytest.raises(exceptions.InvalidProtocolError): SchemaItem(**parameters) + def test_item_construction_default(): - item = SchemaItem(key="a key", label="a label", description="a description", type="a type", displayOrder=2) + item = SchemaItem( + key="a key", + label="a label", + description="a description", + type="a type", + displayOrder=2, + ) assert item.key == "a key" assert item.label == "a label" assert item.description == "a description" @@ -33,13 +47,23 @@ def test_item_construction_default(): assert item.defaultValue == None assert item.widget == None + def test_item_construction_with_optional_params(): - item = SchemaItem(key="a key", label="a label", description="a description", type="a type", displayOrder=2, fileToKeyMap={"file1.txt":"a key"}, defaultValue="some value", widget={}) + item = SchemaItem( + key="a key", + label="a label", + description="a description", + type="a type", + displayOrder=2, + fileToKeyMap={"file1.txt": "a key"}, + defaultValue="some value", + widget={}, + ) assert item.key == "a key" assert item.label == "a label" assert item.description == "a description" assert item.type == "a type" assert item.displayOrder == 2 - assert item.fileToKeyMap == {"file1.txt":"a key"} + assert item.fileToKeyMap == {"file1.txt": "a key"} assert item.defaultValue == "some value" assert item.widget == {} diff --git a/packages/simcore-sdk/tests/unit/test_serialization_v2.py b/packages/simcore-sdk/tests/unit/test_serialization_v2.py new file mode 100644 index 000000000000..3460be302077 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_serialization_v2.py @@ -0,0 +1,47 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + +from typing import Any, Dict + +import pytest +from simcore_sdk.node_ports_v2 import DBManager, exceptions +from simcore_sdk.node_ports_v2.serialization_v2 import dump, load + + +@pytest.mark.parametrize("auto_update", [True, False]) +async def test_load( + mock_db_manager, + auto_update: bool, + node_uuid: str, + default_configuration: Dict[str, Any], +): + db_manager: DBManager = mock_db_manager(default_configuration) + node_ports = await load(db_manager, node_uuid, auto_update) + assert node_ports.db_manager == db_manager + assert node_ports.node_uuid == node_uuid + # pylint: disable=comparison-with-callable + assert node_ports.save_to_db_cb == dump + assert node_ports.node_port_creator_cb == load + assert node_ports.auto_update == auto_update + + +async def test_load_with_invalid_cfg( + mock_db_manager, + node_uuid: str, +): + invalid_config = {"bad_key": "bad_value"} + db_manager: DBManager = mock_db_manager(invalid_config) + with pytest.raises(exceptions.InvalidProtocolError): + _ = await load(db_manager, node_uuid) + + +async def test_dump( + mock_db_manager, + node_uuid: str, + default_configuration: Dict[str, Any], +): + db_manager: DBManager = mock_db_manager(default_configuration) + node_ports = await load(db_manager, node_uuid) + + await dump(node_ports) diff --git a/services/director/src/simcore_service_director/api/v0/schemas/project-v0.0.1.json b/services/director/src/simcore_service_director/api/v0/schemas/project-v0.0.1.json index d61d3cc2107c..a0ba4137bfdd 100644 --- a/services/director/src/simcore_service_director/api/v0/schemas/project-v0.0.1.json +++ b/services/director/src/simcore_service_director/api/v0/schemas/project-v0.0.1.json @@ -210,6 +210,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -302,6 +305,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -344,7 +350,10 @@ ] }, "parent": { - "type": [ "null", "string" ], + "type": [ + "null", + "string" + ], "format": "uuid", "description": "Parent's (group-nodes') node ID s.", "examples": [ diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index cb99877d3030..ea50460cdd02 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -187,3 +187,4 @@ services: - ../packages:/devel/packages environment: - SC_BOOT_MODE=debug-ptvsd + - STORAGE_LOGLEVEL=DEBUG diff --git a/services/sidecar/src/simcore_service_sidecar/core.py b/services/sidecar/src/simcore_service_sidecar/core.py index ba03a26a5c8d..5cf2a2661945 100644 --- a/services/sidecar/src/simcore_service_sidecar/core.py +++ b/services/sidecar/src/simcore_service_sidecar/core.py @@ -12,8 +12,8 @@ comp_pipeline, comp_tasks, ) -from simcore_sdk import node_ports -from simcore_sdk.node_ports import log as node_port_log +from simcore_sdk import node_ports_v2 +from simcore_sdk.node_ports_v2 import log as node_port_v2_log from sqlalchemy import and_, literal_column from . import config, exceptions @@ -24,7 +24,7 @@ log = get_task_logger(__name__) log.setLevel(config.SIDECAR_LOGLEVEL) -node_port_log.setLevel(config.SIDECAR_LOGLEVEL) +node_port_v2_log.setLevel(config.SIDECAR_LOGLEVEL) async def task_required_resources(node_id: str) -> Union[Dict[str, bool], None]: @@ -220,9 +220,9 @@ async def inspect( ) # config nodeports - node_ports.node_config.USER_ID = user_id - node_ports.node_config.NODE_UUID = task.node_id - node_ports.node_config.PROJECT_ID = task.project_id + node_ports_v2.node_config.USER_ID = user_id + node_ports_v2.node_config.NODE_UUID = task.node_id + node_ports_v2.node_config.PROJECT_ID = task.project_id # now proceed actually running the task (we do that after the db session has been closed) # try to run the task, return empyt list of next nodes if anything goes wrong diff --git a/services/sidecar/src/simcore_service_sidecar/executor.py b/services/sidecar/src/simcore_service_sidecar/executor.py index a766c2d8236a..a74d3354592f 100644 --- a/services/sidecar/src/simcore_service_sidecar/executor.py +++ b/services/sidecar/src/simcore_service_sidecar/executor.py @@ -16,8 +16,8 @@ from celery.utils.log import get_task_logger from packaging import version from servicelib.utils import fire_and_forget_task, logged_gather -from simcore_sdk import node_data, node_ports -from simcore_sdk.node_ports.dbmanager import DBManager +from simcore_sdk import node_data, node_ports_v2 +from simcore_sdk.node_ports_v2 import DBManager from . import config, exceptions from .boot_mode import get_boot_mode @@ -146,37 +146,36 @@ async def _get_node_ports(self): if self.db_manager is None: # Keeps single db engine: simcore_sdk.node_ports.dbmanager_{id} self.db_manager = DBManager(self.db_engine) - return await node_ports.ports(self.db_manager) + return await node_ports_v2.ports(self.db_manager) - async def _process_task_input(self, port: node_ports.Port, input_ports: Dict): + async def _process_task_input(self, port: node_ports_v2.Port, input_ports: Dict): + log.debug("getting value from node ports...") port_value = await port.get() - log.debug("PROCESSING %s %s:%s", port.key, type(port_value), port_value) - if str(port.type).startswith("data:"): - path = port_value - if path: - # the filename is not necessarily the name of the port, might be mapped - mapped_filename = Path(path).name - input_ports[port.key] = str(port_value) - final_path = self.shared_folders.input_folder / mapped_filename - shutil.copy(str(path), str(final_path)) - log.debug( - "DOWNLOAD successfull from %s to %s via %s", - port.key, - final_path, - path, - ) - # check if the file is a zip, in that case extract all if the service does not expect a zip file - if zipfile.is_zipfile(final_path) and ( - str(port.type) != "data:application/zip" - ): - with zipfile.ZipFile(final_path, "r") as zip_obj: - zip_obj.extractall(final_path.parents[0]) - # finally remove the zip archive - os.remove(final_path) - else: - input_ports[port.key] = port_value - else: - input_ports[port.key] = port_value + input_ports[port.key] = port_value + log.debug("PROCESSING %s [%s]: %s", port.key, type(port_value), port_value) + if port_value is None: + # we are done here + return + + if isinstance(port_value, Path): + input_ports[port.key] = str(port_value) + # treat files specially, the file shall be moved to the right location + final_path = self.shared_folders.input_folder / port_value.name + shutil.move(port_value, final_path) + log.debug( + "DOWNLOAD successfull from %s to %s via %s", + port.key, + final_path, + port_value, + ) + # check if the file is a zip, in that case extract all if the service does not expect a zip file + if zipfile.is_zipfile(final_path) and ( + str(port.type) != "data:application/zip" + ): + with zipfile.ZipFile(final_path, "r") as zip_obj: + zip_obj.extractall(final_path.parents[0]) + # finally remove the zip archive + os.remove(final_path) async def _process_task_inputs(self) -> Dict: log.debug("Inputs parsing...") @@ -184,7 +183,7 @@ async def _process_task_inputs(self) -> Dict: input_ports: Dict = {} try: PORTS = await self._get_node_ports() - except node_ports.exceptions.NodeNotFound: + except node_ports_v2.exceptions.NodeNotFound: await self._error_message_to_ui_and_logs( "Missing node information in the database" ) @@ -197,7 +196,7 @@ async def _process_task_inputs(self) -> Dict: await logged_gather( *[ self._process_task_input(port, input_ports) - for port in (await PORTS.inputs) + for port in (await PORTS.inputs).values() ] ) log.debug("Inputs parsing DONE") @@ -310,6 +309,7 @@ async def _start_monitoring_container( ) return log_processor_task + # pylint: disable=too-many-statements async def _run_container(self): start_time = time.perf_counter() docker_image = f"{config.DOCKER_REGISTRY}/{self.task.image['name']}:{self.task.image['tag']}" @@ -324,6 +324,8 @@ async def _run_container(self): LogType.LOG, f"[sidecar]Running {self.task.image['name']}:{self.task.image['tag']}...", ) + # ensure progress 0.0 is sent + await self._post_messages(LogType.PROGRESS, "0.0") container = await docker_client.containers.create( config=container_config ) @@ -447,7 +449,7 @@ async def _process_task_output(self): with file_path.open() as fp: output_ports = json.load(fp) task_outputs = await PORTS.outputs - for port in task_outputs: + for port in task_outputs.values(): if port.key in output_ports.keys(): await port.set(output_ports[port.key]) else: @@ -458,7 +460,7 @@ async def _process_task_output(self): # WARNING: nodeports is NOT concurrent-safe, dont' use gather here for coro in file_upload_tasks: await coro - except node_ports.exceptions.NodeNotFound: + except node_ports_v2.exceptions.NodeNotFound: await self._error_message_to_ui_and_logs( "Error: no ports info found in the database." ) @@ -466,7 +468,7 @@ async def _process_task_output(self): await self._error_message_to_ui_and_logs( "Error occurred while decoding output.json" ) - except node_ports.exceptions.NodeportsException: + except node_ports_v2.exceptions.NodeportsException: await self._error_message_to_ui_and_logs( "Error occurred while setting port" ) diff --git a/services/sidecar/src/simcore_service_sidecar/rabbitmq.py b/services/sidecar/src/simcore_service_sidecar/rabbitmq.py index e460e2a16efa..1576c0b14b64 100644 --- a/services/sidecar/src/simcore_service_sidecar/rabbitmq.py +++ b/services/sidecar/src/simcore_service_sidecar/rabbitmq.py @@ -90,6 +90,7 @@ async def close(self): async def _post_message( self, exchange: aio_pika.Exchange, data: Dict[str, Union[str, Any]] ): + log.debug("publishing message to the broker %s", data) await exchange.publish( aio_pika.Message(body=json.dumps(data).encode()), routing_key="" ) diff --git a/services/sidecar/tests/integration/test_sidecar.py b/services/sidecar/tests/integration/test_sidecar.py index e1c67a236df9..d6c1107134d8 100644 --- a/services/sidecar/tests/integration/test_sidecar.py +++ b/services/sidecar/tests/integration/test_sidecar.py @@ -2,6 +2,7 @@ # pylint: disable=redefined-outer-name # pylint: disable=too-many-arguments import asyncio +import importlib import inspect import json from collections import deque @@ -178,7 +179,12 @@ async def _assert_incoming_data_logs( return (sidecar_logs, tasks_logs, progress_logs) -@pytest.fixture +@pytest.fixture( + params=[ + "node_ports", + "node_ports_v2", + ] +) async def pipeline( sidecar_config: None, postgres_host_config: Dict[str, str], @@ -189,16 +195,18 @@ async def pipeline( pipeline_cfg: Dict, mock_dir: Path, user_id: int, + request, ) -> ComputationalPipeline: """creates a full pipeline. NOTE: 'pipeline', defined as parametrization """ - from simcore_sdk import node_ports tasks = {key: osparc_service for key in pipeline_cfg} dag = {key: pipeline_cfg[key]["next"] for key in pipeline_cfg} inputs = {key: pipeline_cfg[key]["inputs"] for key in pipeline_cfg} + np = importlib.import_module(f".{request.param}", package="simcore_sdk") + async def _create( tasks: Dict[str, Any], dag: Dict[str, List[str]], @@ -233,15 +241,15 @@ async def _create( ): # update the files in mock_dir to S3 # FIXME: node_ports config shall not global! here making a hack so it works - node_ports.node_config.USER_ID = user_id - node_ports.node_config.PROJECT_ID = project_id - node_ports.node_config.NODE_UUID = node_uuid + np.node_config.USER_ID = user_id + np.node_config.PROJECT_ID = project_id + np.node_config.NODE_UUID = node_uuid print("--" * 10) - print_module_variables(module=node_ports.node_config) + print_module_variables(module=np.node_config) print("--" * 10) - PORTS = await node_ports.ports() + PORTS = await np.ports() await (await PORTS.inputs)[input_key].set( mock_dir / node_inputs[input_key]["path"] ) @@ -254,29 +262,50 @@ async def _create( "itisfoundation/sleeper", "1.0.0", { - "node_1": { - "next": ["node_2", "node_3"], + "a13d197a-bf8c-4e11-8a15-44a9894cbbe8": { + "next": [ + "28bf052a-5fb8-4935-9c97-2b15109632b9", + "dfdc165b-a10d-4049-bf4e-555bf5e7d557", + ], "inputs": {}, }, - "node_2": { - "next": ["node_4"], + "28bf052a-5fb8-4935-9c97-2b15109632b9": { + "next": ["54901e30-6cd2-417b-aaf9-b458022639d2"], "inputs": { - "in_1": {"nodeUuid": "node_1", "output": "out_1"}, - "in_2": {"nodeUuid": "node_1", "output": "out_2"}, + "in_1": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "out_1", + }, + "in_2": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "out_2", + }, }, }, - "node_3": { - "next": ["node_4"], + "dfdc165b-a10d-4049-bf4e-555bf5e7d557": { + "next": ["54901e30-6cd2-417b-aaf9-b458022639d2"], "inputs": { - "in_1": {"nodeUuid": "node_1", "output": "out_1"}, - "in_2": {"nodeUuid": "node_1", "output": "out_2"}, + "in_1": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "out_1", + }, + "in_2": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "out_2", + }, }, }, - "node_4": { + "54901e30-6cd2-417b-aaf9-b458022639d2": { "next": [], "inputs": { - "in_1": {"nodeUuid": "node_2", "output": "out_1"}, - "in_2": {"nodeUuid": "node_3", "output": "out_2"}, + "in_1": { + "nodeUuid": "28bf052a-5fb8-4935-9c97-2b15109632b9", + "output": "out_1", + }, + "in_2": { + "nodeUuid": "dfdc165b-a10d-4049-bf4e-555bf5e7d557", + "output": "out_2", + }, }, }, }, @@ -286,25 +315,28 @@ async def _create( "itisfoundation/osparc-python-runner", "1.0.0", { - "node_1": { - "next": ["node_2", "node_3"], + "a13d197a-bf8c-4e11-8a15-44a9894cbbe8": { + "next": [ + "28bf052a-5fb8-4935-9c97-2b15109632b9", + "dfdc165b-a10d-4049-bf4e-555bf5e7d557", + ], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_sample.py"} }, }, - "node_2": { - "next": ["node_4"], + "28bf052a-5fb8-4935-9c97-2b15109632b9": { + "next": ["54901e30-6cd2-417b-aaf9-b458022639d2"], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_sample.py"} }, }, - "node_3": { - "next": ["node_4"], + "dfdc165b-a10d-4049-bf4e-555bf5e7d557": { + "next": ["54901e30-6cd2-417b-aaf9-b458022639d2"], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_sample.py"} }, }, - "node_4": { + "54901e30-6cd2-417b-aaf9-b458022639d2": { "next": [], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_sample.py"} @@ -319,18 +351,21 @@ async def _create( "itisfoundation/osparc-python-runner", "1.0.0", { - "node_1": { + "a13d197a-bf8c-4e11-8a15-44a9894cbbe8": { "next": [ - "node_2", + "28bf052a-5fb8-4935-9c97-2b15109632b9", ], "inputs": { "input_1": {"store": SIMCORE_S3_ID, "path": "osparc_python_factory.py"} }, }, - "node_2": { + "28bf052a-5fb8-4935-9c97-2b15109632b9": { "next": [], "inputs": { - "input_1": {"nodeUuid": "node_1", "output": "output_1"}, + "input_1": { + "nodeUuid": "a13d197a-bf8c-4e11-8a15-44a9894cbbe8", + "output": "output_1", + }, }, }, }, diff --git a/services/storage/src/simcore_service_storage/api/v0/openapi.yaml b/services/storage/src/simcore_service_storage/api/v0/openapi.yaml index 16572cd03967..279bc9db0a54 100644 --- a/services/storage/src/simcore_service_storage/api/v0/openapi.yaml +++ b/services/storage/src/simcore_service_storage/api/v0/openapi.yaml @@ -1886,6 +1886,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -1948,6 +1950,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -2315,6 +2319,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -2377,6 +2383,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -2754,6 +2762,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -2816,6 +2826,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -3293,6 +3305,8 @@ components: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -3355,6 +3369,8 @@ components: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: diff --git a/services/storage/src/simcore_service_storage/api/v0/schemas/project-v0.0.1.json b/services/storage/src/simcore_service_storage/api/v0/schemas/project-v0.0.1.json index d61d3cc2107c..a0ba4137bfdd 100644 --- a/services/storage/src/simcore_service_storage/api/v0/schemas/project-v0.0.1.json +++ b/services/storage/src/simcore_service_storage/api/v0/schemas/project-v0.0.1.json @@ -210,6 +210,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -302,6 +305,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -344,7 +350,10 @@ ] }, "parent": { - "type": [ "null", "string" ], + "type": [ + "null", + "string" + ], "format": "uuid", "description": "Parent's (group-nodes') node ID s.", "examples": [ diff --git a/services/storage/src/simcore_service_storage/dsm.py b/services/storage/src/simcore_service_storage/dsm.py index 408166e7c443..7cb09580c36f 100644 --- a/services/storage/src/simcore_service_storage/dsm.py +++ b/services/storage/src/simcore_service_storage/dsm.py @@ -4,10 +4,10 @@ import re import shutil import tempfile +from collections import deque from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Dict, List, Optional, Tuple -from collections import deque import aiobotocore import aiofiles @@ -16,16 +16,14 @@ from aiohttp import web from aiopg.sa import Engine from blackfynn.base import UnauthorizedException -from sqlalchemy.sql import and_ -from tenacity import retry -from yarl import URL - from s3wrapper.s3_client import S3Client from servicelib.aiopg_utils import DBAPIError, PostgresRetryPolicyUponOperation from servicelib.client_session import get_client_session from servicelib.utils import fire_and_forget_task +from sqlalchemy.sql import and_ +from tenacity import retry +from yarl import URL -from .utils import expo from .datcore_wrapper import DatcoreWrapper from .models import ( DatasetMetaData, @@ -46,6 +44,7 @@ SIMCORE_S3_ID, SIMCORE_S3_STR, ) +from .utils import expo # pylint: disable=no-value-for-parameter # FIXME: E1120:No value for argument 'dml' in method call @@ -104,33 +103,33 @@ def to_tuple(self): @attr.s(auto_attribs=True) class DataStorageManager: - """ Data storage manager + """Data storage manager - The dsm has access to the database for all meta data and to the actual backend. For now this - is simcore's S3 [minio] and the datcore storage facilities. + The dsm has access to the database for all meta data and to the actual backend. For now this + is simcore's S3 [minio] and the datcore storage facilities. - For all data that is in-house (simcore.s3, ...) we keep a synchronized database with meta information - for the physical files. + For all data that is in-house (simcore.s3, ...) we keep a synchronized database with meta information + for the physical files. - For physical changes on S3, that might be time-consuming, the db keeps a state (delete and upload mostly) + For physical changes on S3, that might be time-consuming, the db keeps a state (delete and upload mostly) - The dsm provides the following additional functionalities: + The dsm provides the following additional functionalities: - - listing of folders for a given users, optionally filtered using a regular expression and optionally - sorted by one of the meta data keys + - listing of folders for a given users, optionally filtered using a regular expression and optionally + sorted by one of the meta data keys - - upload/download of files + - upload/download of files - client -> S3 : presigned upload link - S3 -> client : presigned download link - datcore -> client: presigned download link - S3 -> datcore: local copy and then upload via their api + client -> S3 : presigned upload link + S3 -> client : presigned download link + datcore -> client: presigned download link + S3 -> datcore: local copy and then upload via their api - minio/S3 and postgres can talk nicely with each other via Notifications using rabbigMQ which we already have. - See: + minio/S3 and postgres can talk nicely with each other via Notifications using rabbigMQ which we already have. + See: - https://blog.minio.io/part-5-5-publish-minio-events-via-postgresql-50f6cc7a7346 - https://docs.minio.io/docs/minio-bucket-notification-guide.html + https://blog.minio.io/part-5-5-publish-minio-events-via-postgresql-50f6cc7a7346 + https://docs.minio.io/docs/minio-bucket-notification-guide.html """ s3_client: S3Client @@ -166,7 +165,7 @@ def location_from_id(cls, location_id: str): return _location_from_id(location_id) async def ping_datcore(self, user_id: str) -> bool: - """ Checks whether user account in datcore is accesible + """Checks whether user account in datcore is accesible :param user_id: user identifier :type user_id: str @@ -193,13 +192,13 @@ async def ping_datcore(self, user_id: str) -> bool: async def list_files( self, user_id: str, location: str, uuid_filter: str = "", regex: str = "" ) -> FileMetaDataExVec: - """ Returns a list of file paths + """Returns a list of file paths - Works for simcore.s3 and datcore + Works for simcore.s3 and datcore - Can filter on uuid: useful to filter on project_id/node_id + Can filter on uuid: useful to filter on project_id/node_id - Can filter upon regular expression (for now only on key: value pairs of the FileMetaData) + Can filter upon regular expression (for now only on key: value pairs of the FileMetaData) """ data = deque() if location == SIMCORE_S3_STR: @@ -308,9 +307,9 @@ async def list_files_dataset( return data async def list_datasets(self, user_id: str, location: str) -> DatasetMetaDataVec: - """ Returns a list of top level datasets + """Returns a list of top level datasets - Works for simcore.s3 and datcore + Works for simcore.s3 and datcore """ data = [] @@ -363,15 +362,15 @@ async def list_file( return data async def delete_file(self, user_id: str, location: str, file_uuid: str): - """ Deletes a file given its fmd and location + """Deletes a file given its fmd and location - Additionally requires a user_id for 3rd party auth + Additionally requires a user_id for 3rd party auth - For internal storage, the db state should be updated upon completion via - Notification mechanism + For internal storage, the db state should be updated upon completion via + Notification mechanism - For simcore.s3 we can use the file_name - For datcore we need the full path + For simcore.s3 we can use the file_name + For datcore we need the full path """ if location == SIMCORE_S3_STR: to_delete = [] @@ -462,13 +461,15 @@ async def metadata_file_updater( await asyncio.sleep(sleep_amount) continue + file_e_tag = result["Contents"][0]["ETag"] # finally update the data in the database and exit continue_loop = False logger.info( - "Obtained this from S3: new_file_size=%s new_last_modified=%s", + "Obtained this from S3: new_file_size=%s new_last_modified=%s file ETag=%s", new_file_size, new_last_modified, + file_e_tag, ) async with self.engine.acquire() as conn: @@ -635,22 +636,22 @@ async def download_link_datcore(self, user_id: str, file_id: str) -> Dict[str, s async def deep_copy_project_simcore_s3( self, user_id: str, source_project, destination_project, node_mapping ): - """ Parses a given source project and copies all related files to the destination project + """Parses a given source project and copies all related files to the destination project - Since all files are organized as + Since all files are organized as - project_id/node_id/filename or links to datcore + project_id/node_id/filename or links to datcore - this function creates a new folder structure + this function creates a new folder structure - project_id/node_id/filename + project_id/node_id/filename - and copies all files to the corresponding places. + and copies all files to the corresponding places. - Additionally, all external files from datcore are being copied and the paths in the destination - project are adapted accordingly + Additionally, all external files from datcore are being copied and the paths in the destination + project are adapted accordingly - Lastly, the meta data db is kept in sync + Lastly, the meta data db is kept in sync """ source_folder = source_project["uuid"] dest_folder = destination_project["uuid"] @@ -773,8 +774,8 @@ async def deep_copy_project_simcore_s3( async def delete_project_simcore_s3( self, user_id: str, project_id: str, node_id: Optional[str] = None ) -> web.Response: - """ Deletes all files from a given node in a project in simcore.s3 and updated db accordingly. - If node_id is not given, then all the project files db entries are deleted. + """Deletes all files from a given node in a project in simcore.s3 and updated db accordingly. + If node_id is not given, then all the project files db entries are deleted. """ async with self.engine.acquire() as conn: diff --git a/services/web/client/source/class/osparc/desktop/WorkbenchView.js b/services/web/client/source/class/osparc/desktop/WorkbenchView.js index 6ea7a662a731..c1734a357cc0 100644 --- a/services/web/client/source/class/osparc/desktop/WorkbenchView.js +++ b/services/web/client/source/class/osparc/desktop/WorkbenchView.js @@ -632,7 +632,7 @@ qx.Class.define("osparc.desktop.WorkbenchView", { socket.on(slotName2, function(data) { const d = JSON.parse(data); const nodeId = d["Node"]; - const progress = 100 * Number.parseFloat(d["Progress"]).toFixed(4); + const progress = Number.parseFloat(d["Progress"]).toFixed(4); const workbench = this.getStudy().getWorkbench(); const node = workbench.getNode(nodeId); if (node) { diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 43f4078df06b..7e1f645bcecb 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -5926,6 +5926,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -5988,6 +5990,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -6486,6 +6490,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -6548,6 +6554,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -6926,6 +6934,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -6988,6 +6998,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -7484,6 +7496,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -7546,6 +7560,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -8048,6 +8064,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -8110,6 +8128,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -8603,6 +8623,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -8665,6 +8687,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -9043,6 +9067,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -9105,6 +9131,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -9623,6 +9651,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -9685,6 +9715,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -11015,6 +11047,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -11077,6 +11111,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -11572,6 +11608,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: @@ -11634,6 +11672,8 @@ paths: type: string label: type: string + eTag: + type: string - type: object additionalProperties: false required: diff --git a/services/web/server/src/simcore_service_webserver/api/v0/schemas/project-v0.0.1.json b/services/web/server/src/simcore_service_webserver/api/v0/schemas/project-v0.0.1.json index d61d3cc2107c..a0ba4137bfdd 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/schemas/project-v0.0.1.json +++ b/services/web/server/src/simcore_service_webserver/api/v0/schemas/project-v0.0.1.json @@ -210,6 +210,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -302,6 +305,9 @@ }, "label": { "type": "string" + }, + "eTag": { + "type": "string" } } }, @@ -344,7 +350,10 @@ ] }, "parent": { - "type": [ "null", "string" ], + "type": [ + "null", + "string" + ], "format": "uuid", "description": "Parent's (group-nodes') node ID s.", "examples": [ diff --git a/services/web/server/tests/integration/computation/test_rabbit.py b/services/web/server/tests/integration/computation/test_rabbit.py index 4e864d342c7b..cd2c44ea2f2c 100644 --- a/services/web/server/tests/integration/computation/test_rabbit.py +++ b/services/web/server/tests/integration/computation/test_rabbit.py @@ -188,7 +188,7 @@ async def _wait_until(pred: Callable, timeout: int): ], ) async def test_rabbit_websocket_computation( - director_v2_subsystem_mock, + director_v2_service_mock, mock_orphaned_services, logged_user, user_project, diff --git a/services/web/server/tests/integration/test_garbage_collection.py b/services/web/server/tests/integration/test_garbage_collection.py index a13409ba3dd6..86b02db3873c 100644 --- a/services/web/server/tests/integration/test_garbage_collection.py +++ b/services/web/server/tests/integration/test_garbage_collection.py @@ -76,7 +76,7 @@ async def __delete_all_redis_keys__(redis_service: RedisConfig): @pytest.fixture -async def director_v2_subsystem_mock() -> aioresponses: +async def director_v2_service_mock() -> aioresponses: """uses aioresponses to mock all calls of an aiohttpclient WARNING: any request done through the client will go through aioresponses. It is unfortunate but that means any valid request (like calling the test server) prefix must be set as passthrough. @@ -103,9 +103,9 @@ async def director_v2_subsystem_mock() -> aioresponses: @pytest.fixture(autouse=True) async def auto_mock_director_v2( - director_v2_subsystem_mock: aioresponses, + director_v2_service_mock: aioresponses, ) -> aioresponses: - return director_v2_subsystem_mock + return director_v2_service_mock @pytest.fixture diff --git a/services/web/server/tests/integration/test_project_workflow.py b/services/web/server/tests/integration/test_project_workflow.py index 994f6fabab85..d8c0d04ad0cd 100644 --- a/services/web/server/tests/integration/test_project_workflow.py +++ b/services/web/server/tests/integration/test_project_workflow.py @@ -250,7 +250,7 @@ async def test_workflow( primary_group: Dict[str, str], standard_groups: List[Dict[str, str]], storage_subsystem_mock, - director_v2_subsystem_mock, + director_v2_service_mock, ): # empty list projects = await _request_list(client) @@ -374,7 +374,7 @@ async def test_list_template_projects( fake_template_projects_isan, fake_template_projects_osparc, catalog_subsystem_mock, - director_v2_subsystem_mock, + director_v2_service_mock, ): catalog_subsystem_mock( fake_template_projects diff --git a/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py b/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py index 3552565d7782..36c63854e7bf 100644 --- a/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py +++ b/services/web/server/tests/unit/with_dbs/fast/test_access_to_studies.py @@ -17,7 +17,6 @@ from aiohttp import ClientResponse, ClientSession, web from aiohttp.test_utils import TestClient from aioresponses import aioresponses - from models_library.projects_state import ( Owner, ProjectLocked, @@ -135,8 +134,8 @@ async def unpublished_project(client, fake_project): @pytest.fixture(autouse=True) -async def director_v2_mock(director_v2_subsystem_mock) -> aioresponses: - yield director_v2_subsystem_mock +async def director_v2_mock(director_v2_service_mock) -> aioresponses: + yield director_v2_service_mock async def _get_user_projects(client): diff --git a/services/web/server/tests/unit/with_dbs/fast/test_director_v2.py b/services/web/server/tests/unit/with_dbs/fast/test_director_v2.py index bb4fe0096c41..76482506db6d 100644 --- a/services/web/server/tests/unit/with_dbs/fast/test_director_v2.py +++ b/services/web/server/tests/unit/with_dbs/fast/test_director_v2.py @@ -5,14 +5,13 @@ from typing import Dict from uuid import UUID, uuid4 -from models_library.projects_state import RunningState -from pydantic.types import PositiveInt import pytest +from _helpers import ExpectedResponse, standard_role_response from aiohttp import web from aioresponses import aioresponses - -from _helpers import ExpectedResponse, standard_role_response +from models_library.projects_state import RunningState +from pydantic.types import PositiveInt from pytest_simcore.helpers.utils_assert import assert_status from pytest_simcore.helpers.utils_login import LoggedUser from simcore_service_webserver import director_v2 @@ -38,9 +37,9 @@ async def logged_user(client, user_role: UserRole): @pytest.fixture(autouse=True) async def auto_mock_director_v2( loop, - director_v2_subsystem_mock: aioresponses, + director_v2_service_mock: aioresponses, ) -> aioresponses: - yield director_v2_subsystem_mock + yield director_v2_service_mock @pytest.fixture diff --git a/services/web/server/tests/unit/with_dbs/medium/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/medium/test_resource_manager.py index 5d229dc64de9..c926be5d4195 100644 --- a/services/web/server/tests/unit/with_dbs/medium/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/medium/test_resource_manager.py @@ -107,8 +107,8 @@ async def empty_user_project2(client, empty_project, logged_user): @pytest.fixture(autouse=True) -async def director_v2_mock(director_v2_subsystem_mock) -> aioresponses: - yield director_v2_subsystem_mock +async def director_v2_mock(director_v2_service_mock) -> aioresponses: + yield director_v2_service_mock # ------------------------ UTILS ---------------------------------- @@ -333,7 +333,7 @@ async def test_interactive_services_removed_after_logout( client_session_id, socketio_client, storage_subsystem_mock, # when guest user logs out garbage is collected - director_v2_subsystem_mock: aioresponses, + director_v2_service_mock: aioresponses, ): set_service_deletion_delay(SERVICE_DELETION_DELAY, client.server.app) # login - logged_user fixture diff --git a/services/web/server/tests/unit/with_dbs/slow/test_projects.py b/services/web/server/tests/unit/with_dbs/slow/test_projects.py index b13f3563d466..19968fb77bcf 100644 --- a/services/web/server/tests/unit/with_dbs/slow/test_projects.py +++ b/services/web/server/tests/unit/with_dbs/slow/test_projects.py @@ -233,9 +233,9 @@ async def mocked_get_services_for_user(*args, **kwargs): @pytest.fixture(autouse=True) async def director_v2_automock( - director_v2_subsystem_mock: aioresponses, + director_v2_service_mock: aioresponses, ) -> aioresponses: - yield director_v2_subsystem_mock + yield director_v2_service_mock # HELPERS ----------------------------------------------------------------------------------------- @@ -525,7 +525,7 @@ async def test_list_projects( template_project, expected, catalog_subsystem_mock, - director_v2_subsystem_mock, + director_v2_service_mock, ): catalog_subsystem_mock([user_project, template_project]) data = await _list_projects(client, expected)