diff --git a/packages/common-library/tests/test_errors_classes.py b/packages/common-library/tests/test_errors_classes.py index efe4c44b86e6..808ed09c40dd 100644 --- a/packages/common-library/tests/test_errors_classes.py +++ b/packages/common-library/tests/test_errors_classes.py @@ -13,20 +13,15 @@ def test_get_full_class_name(): - class A(OsparcErrorMixin): - ... + class A(OsparcErrorMixin): ... - class B1(A): - ... + class B1(A): ... - class B2(A): - ... + class B2(A): ... - class C(B2): - ... + class C(B2): ... - class B12(B1, ValueError): - ... + class B12(B1, ValueError): ... assert B1._get_full_class_name() == "A.B1" assert C._get_full_class_name() == "A.B2.C" diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py new file mode 100644 index 000000000000..5f742464ddda --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/48604dfdc5f4_new_projects_to_job_map.py @@ -0,0 +1,62 @@ +"""new projects to job map + +Revision ID: 48604dfdc5f4 +Revises: 8403acca8759 +Create Date: 2025-03-26 12:00:14.763439+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "48604dfdc5f4" +down_revision = "8403acca8759" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "projects_to_jobs", + sa.Column("project_uuid", sa.String(), nullable=False), + sa.Column( + "job_parent_resource_name", + sa.String(), + nullable=False, + doc="Prefix for the job resource name. For example, if the relative resource name is shelves/shelf1/books/book2, the parent resource name is shelves/shelf1.", + ), + sa.ForeignKeyConstraint( + ["project_uuid"], + ["projects.uuid"], + name="fk_projects_to_jobs_project_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.UniqueConstraint( + "project_uuid", + "job_parent_resource_name", + name="uq_projects_to_jobs_project_uuid_job_parent_resource_name", + ), + ) + + # Populate the new table + op.execute( + sa.text( + r""" +INSERT INTO projects_to_jobs (project_uuid, job_parent_resource_name) +SELECT + uuid AS project_uuid, + regexp_replace(name, '/jobs/.+$', '', 'g') AS job_parent_resource_name -- trim /jobs/.+$ +FROM projects +WHERE name ~* '^solvers/.+/jobs/.+$' OR name ~* '^studies/.+/jobs/.+$'; + """ + ) + ) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("projects_to_jobs") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py new file mode 100644 index 000000000000..4f3859fb36e5 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_to_jobs.py @@ -0,0 +1,37 @@ +import sqlalchemy as sa + +from ._common import RefActions +from .base import metadata +from .projects import projects + +projects_to_jobs = sa.Table( + # Maps projects used as jobs in the public-api + "projects_to_jobs", + metadata, + sa.Column( + "project_uuid", + sa.String, + sa.ForeignKey( + projects.c.uuid, + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_projects_to_jobs_project_uuid", + ), + nullable=False, + doc="Foreign key to projects.uuid", + ), + sa.Column( + "job_parent_resource_name", + sa.String, + nullable=False, + doc="Prefix for the job resource name use in the public-api. For example, if " + "the relative resource name is shelves/shelf1/jobs/job2, " + "the parent resource name is shelves/shelf1.", + ), + # Composite key (project_uuid, job_parent_resource_name) uniquely identifies very row + sa.UniqueConstraint( + "project_uuid", + "job_parent_resource_name", + name="uq_projects_to_jobs_project_uuid_job_parent_resource_name", + ), +) diff --git a/packages/postgres-database/tests/test_models_projects_to_jobs.py b/packages/postgres-database/tests/test_models_projects_to_jobs.py new file mode 100644 index 000000000000..d6f2879694d4 --- /dev/null +++ b/packages/postgres-database/tests/test_models_projects_to_jobs.py @@ -0,0 +1,152 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +from collections.abc import Iterator + +import pytest +import simcore_postgres_database.cli +import sqlalchemy as sa +import sqlalchemy.engine +import sqlalchemy.exc +from faker import Faker +from pytest_simcore.helpers import postgres_tools +from pytest_simcore.helpers.faker_factories import random_project, random_user +from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs +from simcore_postgres_database.models.users import users + + +@pytest.fixture +def sync_engine( + sync_engine: sqlalchemy.engine.Engine, db_metadata: sa.MetaData +) -> Iterator[sqlalchemy.engine.Engine]: + # EXTENDS sync_engine fixture to include cleanup and parare migration + + # cleanup tables + db_metadata.drop_all(sync_engine) + + # prepare migration upgrade + assert simcore_postgres_database.cli.discover.callback + assert simcore_postgres_database.cli.upgrade.callback + + dsn = sync_engine.url + simcore_postgres_database.cli.discover.callback( + user=dsn.username, + password=dsn.password, + host=dsn.host, + database=dsn.database, + port=dsn.port, + ) + + yield sync_engine + + # cleanup tables + postgres_tools.force_drop_all_tables(sync_engine) + + +def test_populate_projects_to_jobs_during_migration( + sync_engine: sqlalchemy.engine.Engine, faker: Faker +): + assert simcore_postgres_database.cli.discover.callback + assert simcore_postgres_database.cli.upgrade.callback + + # UPGRADE just one before 48604dfdc5f4_new_projects_to_job_map.py + simcore_postgres_database.cli.upgrade.callback("8403acca8759") + + with sync_engine.connect() as conn: + + # Ensure the projects_to_jobs table does NOT exist yet + with pytest.raises(sqlalchemy.exc.ProgrammingError) as exc_info: + conn.execute( + sa.select(sa.func.count()).select_from(projects_to_jobs) + ).scalar() + assert "psycopg2.errors.UndefinedTable" in f"{exc_info.value}" + + # INSERT data (emulates data in-place) + user_data = random_user( + faker, name="test_populate_projects_to_jobs_during_migration" + ) + stmt = users.insert().values(**user_data).returning(users.c.id) + result = conn.execute(stmt) + user_id = result.scalar() + + SPACES = " " * 3 + projects_data = [ + random_project( + faker, + uuid="cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + name="solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + description=( + "Study associated to solver job:" + """{ + "id": "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + "name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1/jobs/cd03450c-4c2c-85fd-0d951d7dcd5a", + "inputs_checksum": "015ba4cd5cf00c511a8217deb65c242e3b15dc6ae4b1ecf94982d693887d9e8a", + "created_at": "2025-01-27T13:12:58.676564Z" + } + """ + ), + prj_owner=user_id, + ), + random_project( + faker, + uuid="bf204942-007b-11ef-befd-0242ac114f07", + name=f"studies/4b7a704a-007a-11ef-befd-0242ac114f07/jobs/bf204942-007b-11ef-befd-0242ac114f07{SPACES}", + description="Valid project 2", + prj_owner=user_id, + ), + random_project( + faker, + uuid="33333333-3333-3333-3333-333333333333", + name="invalid/project/name", + description="Invalid project", + prj_owner=user_id, + ), + ] + for prj in projects_data: + conn.execute(sa.insert(projects).values(prj)) + + # MIGRATE UPGRADE: this should populate + simcore_postgres_database.cli.upgrade.callback("head") + + with sync_engine.connect() as conn: + # Query the projects_to_jobs table + result = conn.execute( + sa.select( + projects_to_jobs.c.project_uuid, + projects_to_jobs.c.job_parent_resource_name, + ) + ).fetchall() + + # Assert only valid projects are added + assert len(result) == 2 + assert ( + "cd03450c-4c17-4c2c-85fd-0d951d7dcd5a", + "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.2.1", + ) in result + assert ( + "bf204942-007b-11ef-befd-0242ac114f07", + "studies/4b7a704a-007a-11ef-befd-0242ac114f07", + ) in result + + # Query project name and description for projects also in projects_to_jobs + result = conn.execute( + sa.select( + projects.c.name, + projects.c.uuid, + projects_to_jobs.c.job_parent_resource_name, + ).select_from( + projects.join( + projects_to_jobs, projects.c.uuid == projects_to_jobs.c.project_uuid + ) + ) + ).fetchall() + + # Print or assert the result as needed + for project_name, project_uuid, job_parent_resource_name in result: + assert ( + f"{job_parent_resource_name}/jobs/{project_uuid}" + == project_name.strip() + ) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py new file mode 100644 index 000000000000..bf21eefef1b9 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py @@ -0,0 +1,37 @@ +# pylint: disable=not-context-manager +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +from models_library.products import ProductName +from models_library.projects import ProjectID +from models_library.users import UserID +from pydantic import TypeAdapter, validate_call +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient + + +class WebserverRpcSideEffects: + # pylint: disable=no-self-use + + @validate_call(config={"arbitrary_types_allowed": True}) + async def mark_project_as_job( + self, + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + project_uuid: ProjectID, + job_parent_resource_name: str, + ) -> None: + assert rpc_client + + assert not job_parent_resource_name.startswith("/") # nosec + assert "/" in job_parent_resource_name # nosec + assert not job_parent_resource_name.endswith("/") # nosec + + assert product_name + assert user_id + + TypeAdapter(ProjectID).validate_python(project_uuid) diff --git a/packages/service-library/src/servicelib/common_headers.py b/packages/service-library/src/servicelib/common_headers.py index 543ef593fe59..430823fa776b 100644 --- a/packages/service-library/src/servicelib/common_headers.py +++ b/packages/service-library/src/servicelib/common_headers.py @@ -1,9 +1,9 @@ from typing import Final +UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE: Final[str] = "undefined" X_DYNAMIC_SIDECAR_REQUEST_DNS: Final[str] = "X-Dynamic-Sidecar-Request-DNS" X_DYNAMIC_SIDECAR_REQUEST_SCHEME: Final[str] = "X-Dynamic-Sidecar-Request-Scheme" X_FORWARDED_PROTO: Final[str] = "X-Forwarded-Proto" -X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent" -X_SIMCORE_PARENT_PROJECT_UUID: Final[str] = "X-Simcore-Parent-Project-Uuid" X_SIMCORE_PARENT_NODE_ID: Final[str] = "X-Simcore-Parent-Node-Id" -UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE: Final[str] = "undefined" +X_SIMCORE_PARENT_PROJECT_UUID: Final[str] = "X-Simcore-Parent-Project-Uuid" +X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent" diff --git a/packages/service-library/src/servicelib/rabbitmq/__init__.py b/packages/service-library/src/servicelib/rabbitmq/__init__.py index e2a1416d3e3d..b2e8b6d0b347 100644 --- a/packages/service-library/src/servicelib/rabbitmq/__init__.py +++ b/packages/service-library/src/servicelib/rabbitmq/__init__.py @@ -5,6 +5,7 @@ from ._constants import BIND_TO_ALL_TOPICS, RPC_REQUEST_DEFAULT_TIMEOUT_S from ._errors import ( RemoteMethodNotRegisteredError, + RPCInterfaceError, RPCNotInitializedError, RPCServerError, ) @@ -14,18 +15,19 @@ __all__: tuple[str, ...] = ( "BIND_TO_ALL_TOPICS", + "RPC_REQUEST_DEFAULT_TIMEOUT_S", "ConsumerTag", "ExchangeName", - "is_rabbitmq_responsive", "QueueName", - "RabbitMQClient", - "RabbitMQRPCClient", - "RemoteMethodNotRegisteredError", - "RPC_REQUEST_DEFAULT_TIMEOUT_S", + "RPCInterfaceError", "RPCNamespace", "RPCNotInitializedError", "RPCRouter", "RPCServerError", + "RabbitMQClient", + "RabbitMQRPCClient", + "RemoteMethodNotRegisteredError", + "is_rabbitmq_responsive", "wait_till_rabbitmq_responsive", ) diff --git a/packages/service-library/src/servicelib/rabbitmq/_errors.py b/packages/service-library/src/servicelib/rabbitmq/_errors.py index c105c2b8ff34..ce58b62fd5cb 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/_errors.py @@ -5,17 +5,16 @@ _ERROR_PREFIX: Final[str] = "rabbitmq_error" -class BaseRPCError(OsparcErrorMixin, RuntimeError): - ... +class BaseRPCError(OsparcErrorMixin, RuntimeError): ... class RPCNotInitializedError(BaseRPCError): - code = f"{_ERROR_PREFIX}.not_started" # type: ignore[assignment] + code = f"{_ERROR_PREFIX}.not_started" # type: ignore[assignment] msg_template = "Please check that the RabbitMQ RPC backend was initialized!" class RemoteMethodNotRegisteredError(BaseRPCError): - code = f"{_ERROR_PREFIX}.remote_not_registered" # type: ignore[assignment] + code = f"{_ERROR_PREFIX}.remote_not_registered" # type: ignore[assignment] msg_template = ( "Could not find a remote method named: '{method_name}'. " "Message from remote server was returned: {incoming_message}. " @@ -27,3 +26,23 @@ class RPCServerError(BaseRPCError): "While running method '{method_name}' raised " "'{exc_type}': '{exc_message}'\n{traceback}" ) + + +class RPCInterfaceError(RPCServerError): + """ + Base class for RPC interface exceptions. + + Avoid using domain exceptions directly; if a one-to-one mapping is required, + prefer using the `from_domain_error` transformation function. + """ + + msg_template = "{domain_error_message} [{domain_error_code}]" + + @classmethod + def from_domain_error(cls, err: OsparcErrorMixin): + domain_err_ctx = err.error_context() + return cls( + domain_error_message=domain_err_ctx.pop("message"), + domain_error_code=domain_err_ctx.pop("code"), + **domain_err_ctx, # same context as domain + ) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py new file mode 100644 index 000000000000..e0c3fc2419a2 --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/errors.py @@ -0,0 +1,7 @@ +from ..._errors import RPCInterfaceError + + +class ProjectNotFoundRpcError(RPCInterfaceError): ... + + +class ProjectForbiddenRpcError(RPCInterfaceError): ... diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py new file mode 100644 index 000000000000..f1cafcb16184 --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py @@ -0,0 +1,34 @@ +import logging + +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.products import ProductName +from models_library.projects import ProjectID +from models_library.rabbitmq_basic_types import RPCMethodName +from models_library.users import UserID +from pydantic import TypeAdapter, validate_call +from servicelib.logging_utils import log_decorator +from servicelib.rabbitmq import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +@validate_call(config={"arbitrary_types_allowed": True}) +async def mark_project_as_job( + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + project_uuid: ProjectID, + job_parent_resource_name: str, +) -> None: + + result = await rpc_client.request( + WEBSERVER_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("mark_project_as_job"), + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name=job_parent_resource_name, + ) + assert result is None diff --git a/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc_router.py b/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc_router.py index 4d0c99939ea4..913d5854cd92 100644 --- a/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc_router.py +++ b/packages/service-library/tests/rabbitmq/test_rabbitmq_rpc_router.py @@ -2,12 +2,15 @@ # pylint:disable=unused-argument from collections.abc import Awaitable, Callable +from typing import cast import pytest +from common_library.errors_classes import OsparcErrorMixin from faker import Faker from models_library.rabbitmq_basic_types import RPCMethodName from servicelib.rabbitmq import ( RabbitMQRPCClient, + RPCInterfaceError, RPCNamespace, RPCRouter, RPCServerError, @@ -18,15 +21,21 @@ ] -router = RPCRouter() +class MyServiceError(OsparcErrorMixin, Exception): ... -class MyBaseError(Exception): - ... +class MyDomainError(MyServiceError): + msg_template = "This could happen" -class MyExpectedError(MyBaseError): - ... +def raise_my_expected_error(): + raise MyDomainError(user_id=33, project_id=3) + + +router = RPCRouter() # Server-side + + +class MyExpectedRpcError(RPCInterfaceError): ... @router.expose() @@ -41,10 +50,13 @@ async def an_int_method(a_global_arg: str, *, a_global_kwarg: str) -> int: return 34 -@router.expose(reraise_if_error_type=(MyBaseError,)) -async def raising_expected_error(a_global_arg: str, *, a_global_kwarg: str) -> int: - msg = "This could happen" - raise MyExpectedError(msg) +@router.expose(reraise_if_error_type=(MyExpectedRpcError,)) +async def raising_expected_error(a_global_arg: str, *, a_global_kwarg: str): + try: + raise_my_expected_error() + except MyDomainError as exc: + # NOTE how it is adapted from a domain exception to an interface exception + raise MyExpectedRpcError.from_domain_error(exc) from exc @router.expose() @@ -55,7 +67,7 @@ async def raising_unexpected_error(a_global_arg: str, *, a_global_kwarg: str) -> @pytest.fixture def router_namespace(faker: Faker) -> RPCNamespace: - return faker.pystr() + return cast(RPCNamespace, faker.pystr()) async def test_exposed_methods( @@ -100,10 +112,18 @@ async def test_exposed_methods( assert "builtins.ValueError" in f"{exc_info.value}" # This error was classified int he interface - with pytest.raises(MyBaseError) as exc_info: + with pytest.raises(RPCInterfaceError) as exc_info: await rpc_client.request( router_namespace, RPCMethodName(raising_expected_error.__name__), ) - assert isinstance(exc_info.value, MyExpectedError) + assert isinstance(exc_info.value, MyExpectedRpcError) + assert exc_info.value.error_context() == { + "message": "This could happen [MyServiceError.MyDomainError]", + "code": "RuntimeError.BaseRPCError.RPCServerError", + "domain_error_message": "This could happen", + "domain_error_code": "MyServiceError.MyDomainError", + "user_id": 33, + "project_id": 3, + } diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 609bf0b56a6f..b0737356e7f4 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -34,10 +34,14 @@ create_jobstatus_from_task, create_new_project_for_job, ) +from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.application import get_reverse_url_mapper from ..dependencies.authentication import get_current_user_id, get_product_name from ..dependencies.services import get_api_client from ..dependencies.webserver_http import AuthSession, get_webserver_session +from ..dependencies.webserver_rpc import ( + get_wb_api_rpc_client, +) from ._constants import ( FMSG_CHANGELOG_ADDED_IN_VERSION, FMSG_CHANGELOG_CHANGED_IN_VERSION, @@ -95,6 +99,7 @@ async def create_job( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], catalog_client: Annotated[CatalogApi, Depends(get_api_client(CatalogApi))], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], hidden: Annotated[bool, Query()] = True, @@ -125,6 +130,7 @@ async def create_job( parent_project_uuid=x_simcore_parent_project_uuid, parent_node_id=x_simcore_parent_node_id, ) + assert new_project # nosec assert new_project.uuid == pre_job.id # nosec @@ -139,6 +145,13 @@ async def create_job( assert job.name == pre_job.name # nosec assert job.name == _compose_job_resource_name(solver_key, version, job.id) # nosec + await wb_api_rpc.mark_project_as_job( + product_name=product_name, + user_id=user_id, + project_uuid=new_project.uuid, + job_parent_resource_name=job.runner_name, + ) + return job diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index f1e5513414bc..0043b5daa705 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -16,8 +16,6 @@ from pydantic import PositiveInt from servicelib.logging_utils import log_context -from ...api.dependencies.authentication import get_current_user_id -from ...api.dependencies.services import get_api_client from ...exceptions.backend_errors import ProjectAlreadyStartedError from ...models.pagination import Page, PaginationParams from ...models.schemas.errors import ErrorGet @@ -46,8 +44,14 @@ get_project_and_file_inputs_from_job_inputs, ) from ...services_http.webserver import AuthSession +from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.application import get_reverse_url_mapper +from ..dependencies.authentication import get_current_user_id, get_product_name +from ..dependencies.services import get_api_client from ..dependencies.webserver_http import get_webserver_session +from ..dependencies.webserver_rpc import ( + get_wb_api_rpc_client, +) from ._common import API_SERVER_DEV_FEATURES_ENABLED from ._constants import FMSG_CHANGELOG_CHANGED_IN_VERSION, FMSG_CHANGELOG_NEW_IN_VERSION from .solvers_jobs import JOBS_STATUS_CODES @@ -86,7 +90,10 @@ async def create_study_job( study_id: StudyID, job_inputs: JobInputs, webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + product_name: Annotated[str, Depends(get_product_name)], hidden: Annotated[bool, Query()] = True, x_simcore_parent_project_uuid: ProjectID | None = Header(default=None), x_simcore_parent_node_id: NodeID | None = Header(default=None), @@ -120,6 +127,13 @@ async def create_study_job( patch_params=ProjectPatch(name=job.name), # type: ignore[arg-type] ) + await wb_api_rpc.mark_project_as_job( + product_name=product_name, + user_id=user_id, + project_uuid=job.id, + job_parent_resource_name=job.runner_name, + ) + project_inputs = await webserver_api.get_project_inputs(project_id=project.uuid) file_param_nodes = {} diff --git a/services/api-server/src/simcore_service_api_server/models/api_resources.py b/services/api-server/src/simcore_service_api_server/models/api_resources.py index 9a2221034ad6..1f3e4e71e38c 100644 --- a/services/api-server/src/simcore_service_api_server/models/api_resources.py +++ b/services/api-server/src/simcore_service_api_server/models/api_resources.py @@ -5,7 +5,7 @@ from pydantic import Field, TypeAdapter from pydantic.types import StringConstraints -# RESOURCE NAMES https://cloud.google.com/apis/design/resource_names +# RESOURCE NAMES https://google.aip.dev/122 # # # API Service Name Collection ID Resource ID Collection ID Resource ID @@ -56,3 +56,23 @@ def compose_resource_name(*collection_or_resource_ids) -> RelativeResourceName: def split_resource_name(resource_name: RelativeResourceName) -> list[str]: quoted_parts = resource_name.split("/") return [f"{urllib.parse.unquote_plus(p)}" for p in quoted_parts] + + +def split_resource_name_as_dict( + resource_name: RelativeResourceName, +) -> dict[str, str | None]: + """Returns a map with + resource_ids[Collection-ID] == Resource-ID + """ + parts = split_resource_name(resource_name) + return dict(zip(parts[::2], parts[1::2], strict=False)) + + +def parse_collections_ids(resource_name: RelativeResourceName) -> list[str]: + parts = split_resource_name(resource_name) + return parts[::2] + + +def parse_resources_ids(resource_name: RelativeResourceName) -> list[str]: + parts = split_resource_name(resource_name) + return parts[1::2] diff --git a/services/api-server/src/simcore_service_api_server/services_http/webserver.py b/services/api-server/src/simcore_service_api_server/services_http/webserver.py index 3f9d93c7455c..8b84ba1af6c5 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services_http/webserver.py @@ -1,6 +1,5 @@ # pylint: disable=too-many-public-methods -import json import logging import urllib.parse from collections.abc import Mapping @@ -10,6 +9,7 @@ from uuid import UUID import httpx +from common_library.json_serialization import json_dumps from cryptography import fernet from fastapi import FastAPI, status from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet @@ -193,9 +193,9 @@ async def _page_projects( optional: dict[str, Any] = {} if search_by_project_name is not None: - filters_dict = {"search_by_project_name": search_by_project_name} - filters_json = json.dumps(filters_dict) - optional["filters"] = filters_json + optional["filters"] = json_dumps( + {"search_by_project_name": search_by_project_name} + ) with service_exception_handler( service_name="Webserver", @@ -298,14 +298,16 @@ async def create_project( parent_node_id: NodeID | None, ) -> ProjectGet: # POST /projects --> 202 Accepted - _headers = { + query_params = {"hidden": is_hidden} + headers = { X_SIMCORE_PARENT_PROJECT_UUID: parent_project_uuid, X_SIMCORE_PARENT_NODE_ID: parent_node_id, } + response = await self.client.post( "/projects", - params={"hidden": is_hidden}, - headers={k: f"{v}" for k, v in _headers.items() if v is not None}, + params=query_params, + headers={k: f"{v}" for k, v in headers.items() if v is not None}, json=jsonable_encoder(project, by_alias=True, exclude={"state"}), cookies=self.session_cookies, ) @@ -322,7 +324,8 @@ async def clone_project( parent_project_uuid: ProjectID | None, parent_node_id: NodeID | None, ) -> ProjectGet: - query = {"from_study": project_id, "hidden": hidden} + # POST /projects --> 202 Accepted + query_params = {"from_study": project_id, "hidden": hidden} _headers = { X_SIMCORE_PARENT_PROJECT_UUID: parent_project_uuid, X_SIMCORE_PARENT_NODE_ID: parent_node_id, @@ -331,7 +334,7 @@ async def clone_project( response = await self.client.post( "/projects", cookies=self.session_cookies, - params=query, + params=query_params, headers={k: f"{v}" for k, v in _headers.items() if v is not None}, ) response.raise_for_status() @@ -352,6 +355,8 @@ async def get_project(self, *, project_id: UUID) -> ProjectGet: async def get_projects_w_solver_page( self, *, solver_name: str, limit: int, offset: int ) -> Page[ProjectGet]: + assert not solver_name.endswith("/") # nosec + return await self._page_projects( limit=limit, offset=offset, diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 3781de9e4b77..ca771d913b19 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -6,6 +6,8 @@ from fastapi_pagination import create_page from models_library.api_schemas_webserver.licensed_items import LicensedItemRpcGetPage from models_library.licenses import LicensedItemID +from models_library.products import ProductName +from models_library.projects import ProjectID from models_library.resource_tracker_licensed_items_checkouts import ( LicensedItemCheckoutID, ) @@ -26,6 +28,7 @@ from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( NotEnoughAvailableSeatsError, ) +from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc from servicelib.rabbitmq.rpc_interfaces.webserver.functions.functions import ( ping as _ping, ) @@ -41,6 +44,7 @@ from servicelib.rabbitmq.rpc_interfaces.webserver.licenses.licensed_items import ( release_licensed_item_for_wallet as _release_licensed_item_for_wallet, ) +from simcore_service_api_server.models.api_resources import RelativeResourceName from ..exceptions.backend_errors import ( CanNotCheckoutServiceIsNotRunningError, @@ -93,7 +97,7 @@ class WbApiRpcClient(SingletonInAppStateMixin): @_exception_mapper(rpc_exception_map={}) async def get_licensed_items( - self, *, product_name: str, page_params: PaginationParams + self, *, product_name: ProductName, page_params: PaginationParams ) -> Page[LicensedItemGet]: licensed_items_page = await _get_licensed_items( rabbitmq_rpc_client=self._client, @@ -109,7 +113,7 @@ async def get_licensed_items( async def get_available_licensed_items_for_wallet( self, *, - product_name: str, + product_name: ProductName, wallet_id: WalletID, user_id: UserID, page_params: PaginationParams, @@ -137,7 +141,7 @@ async def get_available_licensed_items_for_wallet( async def checkout_licensed_item_for_wallet( self, *, - product_name: str, + product_name: ProductName, user_id: UserID, wallet_id: WalletID, licensed_item_id: LicensedItemID, @@ -174,7 +178,7 @@ async def checkout_licensed_item_for_wallet( async def release_licensed_item_for_wallet( self, *, - product_name: str, + product_name: ProductName, user_id: UserID, licensed_item_checkout_id: LicensedItemCheckoutID, ) -> LicensedItemCheckoutGet: @@ -200,6 +204,21 @@ async def release_licensed_item_for_wallet( async def ping(self) -> str: return await _ping(self._client) + async def mark_project_as_job( + self, + product_name: ProductName, + user_id: UserID, + project_uuid: ProjectID, + job_parent_resource_name: RelativeResourceName, + ): + await projects_rpc.mark_project_as_job( + rpc_client=self._client, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name=job_parent_resource_name, + ) + def setup(app: FastAPI, rabbitmq_rmp_client: RabbitMQRPCClient): wb_api_rpc_client = WbApiRpcClient(_client=rabbitmq_rmp_client) diff --git a/services/api-server/tests/unit/_with_db/test_api_user.py b/services/api-server/tests/unit/_with_db/test_api_user.py index 93a3bdf8f68b..5b29f72ef158 100644 --- a/services/api-server/tests/unit/_with_db/test_api_user.py +++ b/services/api-server/tests/unit/_with_db/test_api_user.py @@ -10,6 +10,7 @@ import respx from fastapi import FastAPI from models_library.api_schemas_webserver.users import MyProfileGet as WebProfileGet +from pytest_mock import MockType from respx import MockRouter from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.core.settings import ApplicationSettings @@ -18,7 +19,7 @@ @pytest.fixture -def mocked_webserver_service_api(app: FastAPI): +def mocked_webserver_rest_api(app: FastAPI): """Mocks some responses of web-server service""" settings: ApplicationSettings = app.state.settings @@ -53,7 +54,8 @@ def _update_me(request: httpx.Request): async def test_get_profile( client: httpx.AsyncClient, auth: httpx.BasicAuth, - mocked_webserver_service_api: MockRouter, + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], ): # needs no auth resp = await client.get(f"/{API_VTAG}/meta") @@ -62,11 +64,11 @@ async def test_get_profile( # needs auth resp = await client.get(f"/{API_VTAG}/me") assert resp.status_code == status.HTTP_401_UNAUTHORIZED - assert not mocked_webserver_service_api["get_me"].called + assert not mocked_webserver_rest_api["get_me"].called resp = await client.get(f"/{API_VTAG}/me", auth=auth) assert resp.status_code == status.HTTP_200_OK - assert mocked_webserver_service_api["get_me"].called + assert mocked_webserver_rest_api["get_me"].called profile = Profile(**resp.json()) assert profile.first_name == "James" @@ -76,7 +78,8 @@ async def test_get_profile( async def test_update_profile( client: httpx.AsyncClient, auth: httpx.BasicAuth, - mocked_webserver_service_api: MockRouter, + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], ): # needs auth resp = await client.put( diff --git a/services/api-server/tests/unit/_with_db/test_product.py b/services/api-server/tests/unit/_with_db/test_product.py index 274869d094a4..acf209496181 100644 --- a/services/api-server/tests/unit/_with_db/test_product.py +++ b/services/api-server/tests/unit/_with_db/test_product.py @@ -26,7 +26,7 @@ async def test_product_webserver( client: httpx.AsyncClient, - mocked_webserver_service_api_base: respx.MockRouter, + mocked_webserver_rest_api_base: respx.MockRouter, create_fake_api_keys: Callable[[PositiveInt], AsyncGenerator[ApiKeyInDB, None]], faker: Faker, ) -> None: @@ -64,7 +64,7 @@ def _check_key_product_compatibility(request: httpx.Request, **kwargs): ), ) - wallet_get_mock = mocked_webserver_service_api_base.get( + wallet_get_mock = mocked_webserver_rest_api_base.get( path__regex=r"/wallets/(?P[-+]?\d+)" ).mock(side_effect=_check_key_product_compatibility) @@ -80,7 +80,7 @@ def _check_key_product_compatibility(request: httpx.Request, **kwargs): async def test_product_catalog( client: httpx.AsyncClient, - mocked_catalog_service_api_base: respx.MockRouter, + mocked_catalog_rest_api_base: respx.MockRouter, create_fake_api_keys: Callable[[PositiveInt], AsyncGenerator[ApiKeyInDB, None]], ) -> None: assert client @@ -99,7 +99,7 @@ def _get_service_side_effect(request: httpx.Request, **kwargs): assert key.product_name == received_product return httpx.Response(status_code=status.HTTP_200_OK) - respx_mock = mocked_catalog_service_api_base.get( + respx_mock = mocked_catalog_rest_api_base.get( r"/v0/services/simcore%2Fservices%2Fcomp%2Fisolve/2.0.24" ).mock(side_effect=_get_service_side_effect) diff --git a/services/api-server/tests/unit/api_solvers/conftest.py b/services/api-server/tests/unit/api_solvers/conftest.py index 82c125139b63..7c91f7a13a86 100644 --- a/services/api-server/tests/unit/api_solvers/conftest.py +++ b/services/api-server/tests/unit/api_solvers/conftest.py @@ -30,26 +30,26 @@ def solver_version() -> str: @pytest.fixture -def mocked_webserver_service_api( +def mocked_webserver_rest_api( app: FastAPI, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, patch_webserver_long_running_project_tasks: Callable[[MockRouter], MockRouter], ) -> MockRouter: settings: ApplicationSettings = app.state.settings assert settings.API_SERVER_WEBSERVER - patch_webserver_long_running_project_tasks(mocked_webserver_service_api_base) + patch_webserver_long_running_project_tasks(mocked_webserver_rest_api_base) - return mocked_webserver_service_api_base + return mocked_webserver_rest_api_base @pytest.fixture -def mocked_catalog_service_api( +def mocked_catalog_rest_api( app: FastAPI, - mocked_catalog_service_api_base: MockRouter, + mocked_catalog_rest_api_base: MockRouter, catalog_service_openapi_specs: dict[str, Any], ) -> MockRouter: - respx_mock = mocked_catalog_service_api_base + respx_mock = mocked_catalog_rest_api_base openapi = deepcopy(catalog_service_openapi_specs) schemas = openapi["components"]["schemas"] @@ -90,8 +90,8 @@ def mocked_catalog_service_api( @pytest.fixture -async def mocked_directorv2_service( - mocked_directorv2_service_api_base, +async def mocked_directorv2_rest_api( + mocked_directorv2_rest_api_base, ) -> AsyncIterable[MockRouter]: stop_time: Final[datetime] = datetime.now() + timedelta(seconds=5) @@ -106,7 +106,7 @@ def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: status_code=status.HTTP_200_OK, json=jsonable_encoder(task) ) - mocked_directorv2_service_api_base.get( + mocked_directorv2_rest_api_base.get( path__regex=r"/v2/computations/(?P[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" ).mock(side_effect=_get_computation) - return mocked_directorv2_service_api_base + return mocked_directorv2_rest_api_base diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py index d4a6cf80a76e..c07c725fd760 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py @@ -18,7 +18,7 @@ @pytest.mark.skip(reason="Still under development. Currently using fake implementation") async def test_list_solvers( client: httpx.AsyncClient, - mocked_catalog_service_api: MockRouter, + mocked_catalog_rest_api: MockRouter, mocker: MockFixture, ): warn = mocker.patch.object( @@ -63,7 +63,7 @@ async def test_list_solvers( async def test_list_solver_ports( - mocked_catalog_service_api: MockRouter, + mocked_catalog_rest_api: MockRouter, client: httpx.AsyncClient, auth: httpx.BasicAuth, ): diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py index e3f97def12d1..a167f2b4603d 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs.py @@ -18,6 +18,7 @@ from models_library.services import ServiceMetaDataPublished from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import AnyUrl, HttpUrl, TypeAdapter +from pytest_mock import MockType from respx import MockRouter from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.core.settings import ApplicationSettings @@ -85,7 +86,7 @@ def presigned_download_link( def mocked_directorv2_service_api( app: FastAPI, presigned_download_link: AnyUrl, - mocked_directorv2_service_api_base: MockRouter, + mocked_directorv2_rest_api_base: MockRouter, directorv2_service_openapi_specs: dict[str, Any], ): settings: ApplicationSettings = app.state.settings @@ -93,7 +94,7 @@ def mocked_directorv2_service_api( oas = directorv2_service_openapi_specs # pylint: disable=not-context-manager - respx_mock = mocked_directorv2_service_api_base + respx_mock = mocked_directorv2_rest_api_base # check that what we emulate, actually still exists path = "/v2/computations/{project_id}/tasks/-/logfile" assert path in oas["paths"] @@ -203,9 +204,10 @@ async def test_run_solver_job( client: httpx.AsyncClient, directorv2_service_openapi_specs: dict[str, Any], catalog_service_openapi_specs: dict[str, Any], - mocked_catalog_service_api: MockRouter, + mocked_catalog_rest_api: MockRouter, mocked_directorv2_service_api: MockRouter, - mocked_webserver_service_api: MockRouter, + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], auth: httpx.BasicAuth, project_id: str, solver_key: str, @@ -278,7 +280,7 @@ async def test_run_solver_job( ), ) - mocked_webserver_service_api.post( + mocked_webserver_rest_api.post( path__regex=r"^/v0/computations/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-(3|4|5)[0-9a-fA-F]{3}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}:start$", name="webserver_start_job", ).respond( @@ -317,7 +319,7 @@ async def test_run_solver_job( if "boot-options" in e ) - mocked_catalog_service_api.get( + mocked_catalog_rest_api.get( # path__regex=r"/services/(?P[\w-]+)/(?P[0-9\.]+)", path=f"/v0/services/{solver_key}/{solver_version}", name="get_service_v0_services__service_key___service_version__get", @@ -353,9 +355,9 @@ async def test_run_solver_job( ) assert resp.status_code == status.HTTP_201_CREATED - assert mocked_webserver_service_api["create_projects"].called - assert mocked_webserver_service_api["get_task_status"].called - assert mocked_webserver_service_api["get_task_result"].called + assert mocked_webserver_rest_api["create_projects"].called + assert mocked_webserver_rest_api["get_task_status"].called + assert mocked_webserver_rest_api["get_task_result"].called job = Job.model_validate(resp.json()) diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py index bdb3886ebecd..c5fad3f6d2ae 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_delete.py @@ -12,6 +12,7 @@ from faker import Faker from models_library.basic_regex import UUID_RE_BASE from pydantic import TypeAdapter +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from servicelib.common_headers import ( @@ -32,7 +33,8 @@ class MockedBackendApiDict(TypedDict): @pytest.fixture def mocked_backend_services_apis_for_delete_non_existing_project( - mocked_webserver_service_api: MockRouter, + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "delete_project_not_found.json" @@ -49,12 +51,12 @@ def _response(request: httpx.Request, project_id: str): status_code=capture.status_code, json=capture.response_body ) - mocked_webserver_service_api.delete( + mocked_webserver_rest_api.delete( path__regex=rf"/projects/(?P{UUID_RE_BASE})$", name="delete_project", ).mock(side_effect=_response) - return MockedBackendApiDict(webserver=mocked_webserver_service_api, catalog=None) + return MockedBackendApiDict(webserver=mocked_webserver_rest_api, catalog=None) @pytest.mark.acceptance_test( @@ -84,8 +86,9 @@ async def test_delete_non_existing_solver_job( @pytest.fixture def mocked_backend_services_apis_for_create_and_delete_solver_job( - mocked_webserver_service_api: MockRouter, - mocked_catalog_service_api: MockRouter, + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_catalog_rest_api: MockRouter, project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "on_create_job.json" @@ -98,7 +101,7 @@ def mocked_backend_services_apis_for_create_and_delete_solver_job( capture = captures[0] assert capture.host == "catalog" assert capture.method == "GET" - mocked_catalog_service_api.request( + mocked_catalog_rest_api.request( method=capture.method, path=capture.path, name="get_service" # GET service ).respond(status_code=capture.status_code, json=capture.response_body) @@ -106,14 +109,14 @@ def mocked_backend_services_apis_for_create_and_delete_solver_job( assert capture.host == "webserver" assert capture.method == "DELETE" - mocked_webserver_service_api.delete( + mocked_webserver_rest_api.delete( path__regex=rf"/projects/(?P{UUID_RE_BASE})$", name="delete_project", ).respond(status_code=capture.status_code, json=capture.response_body) return MockedBackendApiDict( - catalog=mocked_catalog_service_api, - webserver=mocked_webserver_service_api, + catalog=mocked_catalog_rest_api, + webserver=mocked_webserver_rest_api, ) diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py index 80a7176ca85d..901eea6277c9 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_logs.py @@ -98,7 +98,7 @@ async def test_log_streaming( solver_version: str, fake_log_distributor, fake_project_for_streaming: ProjectGet, - mocked_directorv2_service: MockRouter, + mocked_directorv2_rest_api: MockRouter, disconnect: bool, ): @@ -127,15 +127,15 @@ async def test_log_streaming( @pytest.fixture async def mock_job_not_found( - mocked_directorv2_service_api_base: MockRouter, + mocked_directorv2_rest_api_base: MockRouter, ) -> MockRouter: def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: return httpx.Response(status_code=status.HTTP_404_NOT_FOUND) - mocked_directorv2_service_api_base.get( + mocked_directorv2_rest_api_base.get( path__regex=r"/v2/computations/(?P[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})" ).mock(side_effect=_get_computation) - return mocked_directorv2_service_api_base + return mocked_directorv2_rest_api_base async def test_logstreaming_job_not_found_exception( diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py index 6b62c89b6b87..1693e579fa8b 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_metadata.py @@ -11,6 +11,7 @@ from faker import Faker from models_library.basic_regex import UUID_RE_BASE from pydantic import TypeAdapter +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from simcore_service_api_server._meta import API_VTAG @@ -37,8 +38,9 @@ def _as_path_regex(initial_path: str): @pytest.fixture def mocked_backend( - mocked_webserver_service_api: MockRouter, - mocked_catalog_service_api: MockRouter, + mocked_webserver_rest_api: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_catalog_rest_api: MockRouter, project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "for_test_get_and_update_job_metadata.json" @@ -52,7 +54,7 @@ def mocked_backend( capture = captures["get_service"] assert capture.host == "catalog" - mocked_catalog_service_api.request( + mocked_catalog_rest_api.request( method=capture.method, path=capture.path, name=capture.name, @@ -66,7 +68,7 @@ def mocked_backend( assert capture.host == "webserver" capture_path_regex = _as_path_regex(capture.path.removeprefix("/v0")) - route = mocked_webserver_service_api.request( + route = mocked_webserver_rest_api.request( method=capture.method, path__regex=capture_path_regex, name=capture.name, @@ -86,7 +88,7 @@ def mocked_backend( ) return MockedBackendApiDict( - webserver=mocked_webserver_service_api, catalog=mocked_catalog_service_api + webserver=mocked_webserver_rest_api, catalog=mocked_catalog_rest_api ) diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py index b51c580eb822..09fc53194f62 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py @@ -23,8 +23,8 @@ class MockBackendRouters(NamedTuple): @pytest.fixture def mocked_backend( - mocked_webserver_service_api_base: MockRouter, - mocked_catalog_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, + mocked_catalog_rest_api_base: MockRouter, project_tests_dir: Path, ) -> MockBackendRouters: mock_name = "on_list_jobs.json" @@ -35,7 +35,7 @@ def mocked_backend( capture = captures[0] assert capture.host == "catalog" assert capture.name == "get_service" - mocked_catalog_service_api_base.request( + mocked_catalog_rest_api_base.request( method=capture.method, path=capture.path, name=capture.name, @@ -47,7 +47,7 @@ def mocked_backend( capture = captures[1] assert capture.host == "webserver" assert capture.name == "list_projects" - mocked_webserver_service_api_base.request( + mocked_webserver_rest_api_base.request( method=capture.method, name=capture.name, path=capture.path, @@ -57,8 +57,8 @@ def mocked_backend( ) return MockBackendRouters( - catalog=mocked_catalog_service_api_base, - webserver=mocked_webserver_service_api_base, + catalog=mocked_catalog_rest_api_base, + webserver=mocked_webserver_rest_api_base, ) diff --git a/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py b/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py index d1fae307589d..b1cc4c800735 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py +++ b/services/api-server/tests/unit/api_studies/test_api_routers_studies_jobs_metadata.py @@ -13,6 +13,7 @@ import pytest from fastapi.encoders import jsonable_encoder from pydantic import TypeAdapter +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from pytest_simcore.helpers.httpx_calls_capture_parameters import PathDescription from respx import MockRouter @@ -33,7 +34,8 @@ class MockedBackendApiDict(TypedDict): @pytest.fixture def mocked_backend( project_tests_dir: Path, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], ) -> MockedBackendApiDict | None: # load captures = { @@ -85,7 +87,7 @@ def mocked_backend( if group: # mock this entrypoint using https://lundberg.github.io/respx/guide/#iterable cc = [c] + [captures[_] for _ in group] - mocked_webserver_service_api_base.request( + mocked_webserver_rest_api_base.request( method=c.method.upper(), url=None, path__regex=f"^{c.path.to_path_regex()}$", @@ -94,7 +96,7 @@ def mocked_backend( side_effect=[_.as_response() for _ in cc], ) else: - mocked_webserver_service_api_base.request( + mocked_webserver_rest_api_base.request( method=c.method.upper(), url=None, path__regex=f"^{c.path.to_path_regex()}$", @@ -102,7 +104,7 @@ def mocked_backend( ).mock(return_value=c.as_response()) return MockedBackendApiDict( - webserver=mocked_webserver_service_api_base, + webserver=mocked_webserver_rest_api_base, ) diff --git a/services/api-server/tests/unit/api_studies/test_api_routes_studies.py b/services/api-server/tests/unit/api_studies/test_api_routes_studies.py index d5369bb0314a..6c289763d7bb 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routes_studies.py +++ b/services/api-server/tests/unit/api_studies/test_api_routes_studies.py @@ -13,6 +13,7 @@ from faker import Faker from fastapi import status from pydantic import TypeAdapter +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from servicelib.common_headers import ( @@ -33,7 +34,8 @@ class MockedBackendApiDict(TypedDict): @pytest.fixture def mocked_backend( - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], project_tests_dir: Path, ) -> MockedBackendApiDict: mock_name = "for_test_api_routes_studies.json" @@ -56,7 +58,7 @@ def mocked_backend( capture = captures[name] assert capture.host == "webserver" - route = mocked_webserver_service_api_base.request( + route = mocked_webserver_rest_api_base.request( method=capture.method, path__regex=capture.path.removeprefix("/v0") + "$", name=capture.name, @@ -65,9 +67,7 @@ def mocked_backend( json=capture.response_body, ) print(route) - return MockedBackendApiDict( - webserver=mocked_webserver_service_api_base, catalog=None - ) + return MockedBackendApiDict(webserver=mocked_webserver_rest_api_base, catalog=None) @pytest.mark.acceptance_test( @@ -124,13 +124,13 @@ async def test_studies_read_workflow( async def test_list_study_ports( client: httpx.AsyncClient, auth: httpx.BasicAuth, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, fake_study_ports: list[dict[str, Any]], study_id: StudyID, ): # Mocks /projects/{*}/metadata/ports - mocked_webserver_service_api_base.get( + mocked_webserver_rest_api_base.get( path__regex=r"/projects/(?P[\w-]+)/metadata/ports$", name="list_project_metadata_ports", ).respond( @@ -155,15 +155,15 @@ async def test_clone_study( client: httpx.AsyncClient, auth: httpx.BasicAuth, study_id: StudyID, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, patch_webserver_long_running_project_tasks: Callable[[MockRouter], MockRouter], parent_project_id: UUID | None, parent_node_id: UUID | None, ): # Mocks /projects - patch_webserver_long_running_project_tasks(mocked_webserver_service_api_base) + patch_webserver_long_running_project_tasks(mocked_webserver_rest_api_base) - callback = mocked_webserver_service_api_base["create_projects"].side_effect + callback = mocked_webserver_rest_api_base["create_projects"].side_effect assert callback is not None def clone_project_side_effect(request: httpx.Request): @@ -179,9 +179,9 @@ def clone_project_side_effect(request: httpx.Request): assert _parent_node_id == f"{parent_node_id}" return callback(request) - mocked_webserver_service_api_base[ - "create_projects" - ].side_effect = clone_project_side_effect + mocked_webserver_rest_api_base["create_projects"].side_effect = ( + clone_project_side_effect + ) _headers = {} if parent_project_id is not None: @@ -192,7 +192,7 @@ def clone_project_side_effect(request: httpx.Request): f"/{API_VTAG}/studies/{study_id}:clone", headers=_headers, auth=auth ) - assert mocked_webserver_service_api_base["create_projects"].called + assert mocked_webserver_rest_api_base["create_projects"].called assert resp.status_code == status.HTTP_201_CREATED @@ -201,11 +201,11 @@ async def test_clone_study_not_found( client: httpx.AsyncClient, auth: httpx.BasicAuth, faker: Faker, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, patch_webserver_long_running_project_tasks: Callable[[MockRouter], MockRouter], ): # Mocks /projects - mocked_webserver_service_api_base.post( + mocked_webserver_rest_api_base.post( path__regex=r"/projects", name="project_clone", ).respond( diff --git a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py index 811818a8939b..84b300cf07ae 100644 --- a/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py +++ b/services/api-server/tests/unit/api_studies/test_api_routes_studies_jobs.py @@ -14,11 +14,11 @@ import respx from faker import Faker from fastapi import status +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import ( CreateRespxMockCallback, HttpApiCallCaptureModel, ) -from respx import MockRouter from servicelib.common_headers import ( X_SIMCORE_PARENT_NODE_ID, X_SIMCORE_PARENT_PROJECT_UUID, @@ -37,7 +37,8 @@ async def test_studies_jobs_workflow( client: httpx.AsyncClient, auth: httpx.BasicAuth, - mocked_webserver_service_api_base: respx.MockRouter, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], study_id: StudyID, ): # get_study @@ -121,8 +122,9 @@ async def test_studies_jobs_workflow( async def test_start_stop_delete_study_job( client: httpx.AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_directorv2_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -154,8 +156,8 @@ def _side_effect_with_project_id( create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, ], capture_path=capture_file, side_effects_callbacks=[_side_effect_no_project_id] @@ -200,8 +202,9 @@ def _check_response(response: httpx.Response, status_code: int): @pytest.mark.parametrize("hidden", [True, False]) async def test_create_study_job( client: httpx.AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_directorv2_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -251,8 +254,8 @@ def _default_side_effect( create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, ], capture_path=_capture_file, side_effects_callbacks=[_default_side_effect] * 5, @@ -279,7 +282,8 @@ async def test_get_study_job_outputs( client: httpx.AsyncClient, fake_study_id: UUID, auth: httpx.BasicAuth, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], ): job_id = "cfe9a77a-f71e-11ee-8fca-0242ac140008" @@ -316,7 +320,7 @@ async def test_get_study_job_outputs( "status_code": 200, } - mocked_webserver_service_api_base.get( + mocked_webserver_rest_api_base.get( path=capture["path"]["path"].format(project_id=job_id) ).respond( status_code=capture["status_code"], @@ -336,8 +340,9 @@ async def test_get_study_job_outputs( async def test_get_job_logs( client: httpx.AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_directorv2_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -347,7 +352,7 @@ async def test_get_job_logs( create_respx_mock_from_capture( respx_mocks=[ - mocked_directorv2_service_api_base, + mocked_directorv2_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "get_study_job_logs.json", side_effects_callbacks=[], @@ -363,8 +368,9 @@ async def test_get_job_logs( async def test_get_study_outputs( client: httpx.AsyncClient, create_respx_mock_from_capture: CreateRespxMockCallback, - mocked_directorv2_service_api_base, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], + mocked_directorv2_rest_api_base: respx.MockRouter, auth: httpx.BasicAuth, project_tests_dir: Path, ): @@ -373,8 +379,8 @@ async def test_get_study_outputs( create_respx_mock_from_capture( respx_mocks=[ - mocked_directorv2_service_api_base, - mocked_webserver_service_api_base, + mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "get_job_outputs.json", side_effects_callbacks=[], diff --git a/services/api-server/tests/unit/api_studies/test_api_studies_mocks.py b/services/api-server/tests/unit/api_studies/test_api_studies_mocks.py index df7f10dc3ce8..4727b0f87f17 100644 --- a/services/api-server/tests/unit/api_studies/test_api_studies_mocks.py +++ b/services/api-server/tests/unit/api_studies/test_api_studies_mocks.py @@ -12,7 +12,7 @@ def test_mocked_webserver_service_api( app: FastAPI, - mocked_webserver_service_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, services_mocks_enabled: bool, ): if not services_mocks_enabled: @@ -34,4 +34,4 @@ def test_mocked_webserver_service_api( assert resp.status_code == status.HTTP_200_OK assert resp.json() - mocked_webserver_service_api_base.assert_all_called() + mocked_webserver_rest_api_base.assert_all_called() diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 83db84a5d2c3..8fb9fb2a4452 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -37,12 +37,14 @@ from moto.server import ThreadedMotoServer from packaging.version import Version from pydantic import EmailStr, HttpUrl, TypeAdapter -from pytest_mock import MockerFixture +from pytest_mock import MockerFixture, MockType from pytest_simcore.helpers.host import get_localhost_ip from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.webserver_rpc_server import WebserverRpcSideEffects from pytest_simcore.simcore_webserver_projects_rest_api import GET_PROJECT from requests.auth import HTTPBasicAuth from respx import MockRouter +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from simcore_service_api_server.core.application import init_app from simcore_service_api_server.core.settings import ApplicationSettings from simcore_service_api_server.db.repositories.api_keys import UserAndProductTuple @@ -256,7 +258,7 @@ def catalog_service_openapi_specs(osparc_simcore_services_dir: Path) -> dict[str @pytest.fixture -def mocked_directorv2_service_api_base( +def mocked_directorv2_rest_api_base( app: FastAPI, directorv2_service_openapi_specs: dict[str, Any], services_mocks_enabled: bool, @@ -291,7 +293,7 @@ def mocked_directorv2_service_api_base( @pytest.fixture -def mocked_webserver_service_api_base( +def mocked_webserver_rest_api_base( app: FastAPI, webserver_service_openapi_specs: dict[str, Any], services_mocks_enabled: bool, @@ -333,7 +335,36 @@ def mocked_webserver_service_api_base( @pytest.fixture -def mocked_storage_service_api_base( +def mocked_webserver_rpc_api( + app: FastAPI, mocker: MockerFixture +) -> dict[str, MockType]: + from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc + from simcore_service_api_server.services_rpc import wb_api_server + + # NOTE: mock_missing_plugins patches `setup_rabbitmq` + try: + wb_api_server.WbApiRpcClient.get_from_app_state(app) + except AttributeError: + wb_api_server.setup( + app, RabbitMQRPCClient("fake_rpc_client", settings=mocker.MagicMock()) + ) + + settings: ApplicationSettings = app.state.settings + assert settings.API_SERVER_WEBSERVER + + side_effects = WebserverRpcSideEffects() + + return { + "mark_project_as_job": mocker.patch.object( + projects_rpc, + "mark_project_as_job", + side_effects.mark_project_as_job, + ), + } + + +@pytest.fixture +def mocked_storage_rest_api_base( app: FastAPI, storage_service_openapi_specs: dict[str, Any], faker: Faker, @@ -393,7 +424,7 @@ def mocked_storage_service_api_base( @pytest.fixture -def mocked_catalog_service_api_base( +def mocked_catalog_rest_api_base( app: FastAPI, catalog_service_openapi_specs: dict[str, Any], services_mocks_enabled: bool, diff --git a/services/api-server/tests/unit/test_api__study_workflows.py b/services/api-server/tests/unit/test_api__study_workflows.py index dc6abbdadaf2..0656ad497328 100644 --- a/services/api-server/tests/unit/test_api__study_workflows.py +++ b/services/api-server/tests/unit/test_api__study_workflows.py @@ -14,10 +14,10 @@ import httpx import pytest +import respx from fastapi.encoders import jsonable_encoder from pytest_mock import MockerFixture from pytest_simcore.helpers.httpx_calls_capture_models import CreateRespxMockCallback -from respx import MockRouter from simcore_sdk.node_ports_common.filemanager import UploadedFile from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.models.pagination import OnePage @@ -196,17 +196,18 @@ def test_py_path(tmp_path: Path) -> Path: class MockedBackendApiDict(TypedDict): - webserver: MockRouter | None - storage: MockRouter | None - director_v2: MockRouter | None + webserver: respx.MockRouter | None + storage: respx.MockRouter | None + director_v2: respx.MockRouter | None @pytest.fixture def mocked_backend( project_tests_dir: Path, - mocked_webserver_service_api_base: MockRouter, - mocked_storage_service_api_base: MockRouter, - mocked_directorv2_service_api_base: MockRouter, + mocked_webserver_rest_api_base: respx.MockRouter, + mocked_webserver_rpc_api: respx.MockRouter, + mocked_storage_rest_api_base: respx.MockRouter, + mocked_directorv2_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, mocker: MockerFixture, ) -> MockedBackendApiDict: @@ -218,17 +219,17 @@ def mocked_backend( create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_storage_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_storage_rest_api_base, + mocked_directorv2_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "run_study_workflow.json", side_effects_callbacks=[], ) return MockedBackendApiDict( - webserver=mocked_webserver_service_api_base, - storage=mocked_storage_service_api_base, - director_v2=mocked_directorv2_service_api_base, + webserver=mocked_webserver_rest_api_base, + storage=mocked_storage_rest_api_base, + director_v2=mocked_directorv2_rest_api_base, ) diff --git a/services/api-server/tests/unit/test_api_files.py b/services/api-server/tests/unit/test_api_files.py index 323332f1f5bd..8f44f00371ee 100644 --- a/services/api-server/tests/unit/test_api_files.py +++ b/services/api-server/tests/unit/test_api_files.py @@ -96,7 +96,7 @@ def checksum(cls) -> SHA256Str: @pytest.mark.xfail(reason="Under dev") async def test_list_files_legacy( - client: AsyncClient, mocked_storage_service_api_base: MockRouter + client: AsyncClient, mocked_storage_rest_api_base: MockRouter ): response = await client.get(f"{API_VTAG}/files") @@ -117,7 +117,7 @@ async def test_list_files_legacy( @pytest.mark.xfail(reason="Under dev") async def test_list_files_with_pagination( client: AsyncClient, - mocked_storage_service_api_base: MockRouter, + mocked_storage_rest_api_base: MockRouter, ): response = await client.get(f"{API_VTAG}/files/page") @@ -146,7 +146,7 @@ async def test_list_files_with_pagination( @pytest.mark.xfail(reason="Under dev") async def test_upload_content( - client: AsyncClient, mocked_storage_service_api_base: MockRouter, tmp_path: Path + client: AsyncClient, mocked_storage_rest_api_base: MockRouter, tmp_path: Path ): upload_path = tmp_path / "test_upload_content.txt" upload_path.write_text("test_upload_content") @@ -166,7 +166,7 @@ async def test_upload_content( @pytest.mark.xfail(reason="Under dev") async def test_get_file( - client: AsyncClient, mocked_storage_service_api_base: MockRouter, tmp_path: Path + client: AsyncClient, mocked_storage_rest_api_base: MockRouter, tmp_path: Path ): response = await client.get( f"{API_VTAG}/files/3fa85f64-5717-4562-b3fc-2c963f66afa6" @@ -183,7 +183,7 @@ async def test_get_file( async def test_delete_file( client: AsyncClient, - mocked_storage_service_api_base: respx.MockRouter, + mocked_storage_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -205,7 +205,7 @@ def delete_side_effect( return capture.response_body create_respx_mock_from_capture( - respx_mocks=[mocked_storage_service_api_base], + respx_mocks=[mocked_storage_rest_api_base], capture_path=project_tests_dir / "mocks" / "delete_file.json", side_effects_callbacks=[search_side_effect, delete_side_effect], ) @@ -218,7 +218,7 @@ def delete_side_effect( @pytest.mark.xfail(reason="Under dev") async def test_download_content( - client: AsyncClient, mocked_storage_service_api_base: MockRouter, tmp_path: Path + client: AsyncClient, mocked_storage_rest_api_base: MockRouter, tmp_path: Path ): response = await client.get( f"{API_VTAG}/files/3fa85f64-5717-4562-b3fc-2c963f66afa6/content" @@ -297,7 +297,7 @@ async def test_get_upload_links( async def test_search_file( query: dict[str, str], client: AsyncClient, - mocked_storage_service_api_base: respx.MockRouter, + mocked_storage_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -325,7 +325,7 @@ def side_effect_callback( return response create_respx_mock_from_capture( - respx_mocks=[mocked_storage_service_api_base], + respx_mocks=[mocked_storage_rest_api_base], capture_path=project_tests_dir / "mocks" / "search_file_checksum.json", side_effects_callbacks=[side_effect_callback], ) diff --git a/services/api-server/tests/unit/test_api_health.py b/services/api-server/tests/unit/test_api_health.py index d421703b0f27..a5d923439624 100644 --- a/services/api-server/tests/unit/test_api_health.py +++ b/services/api-server/tests/unit/test_api_health.py @@ -30,10 +30,10 @@ def healthy(self) -> bool: async def test_get_service_state( client: AsyncClient, - mocked_catalog_service_api_base: MockRouter, - mocked_directorv2_service_api_base: MockRouter, - mocked_storage_service_api_base: MockRouter, - mocked_webserver_service_api_base: MockRouter, + mocked_catalog_rest_api_base: MockRouter, + mocked_directorv2_rest_api_base: MockRouter, + mocked_storage_rest_api_base: MockRouter, + mocked_webserver_rest_api_base: MockRouter, ): response = await client.get(f"{API_VTAG}/state") assert response.status_code == status.HTTP_200_OK diff --git a/services/api-server/tests/unit/test_api_solver_jobs.py b/services/api-server/tests/unit/test_api_solver_jobs.py index 0f0e91f126f4..70171ce27366 100644 --- a/services/api-server/tests/unit/test_api_solver_jobs.py +++ b/services/api-server/tests/unit/test_api_solver_jobs.py @@ -59,7 +59,7 @@ def _inspect_job_side_effect( ) async def test_get_solver_job_wallet( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -94,7 +94,7 @@ def _get_wallet_side_effect( return response create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / capture, side_effects_callbacks=[_get_job_wallet_side_effect, _get_wallet_side_effect], ) @@ -131,7 +131,7 @@ def _get_wallet_side_effect( ) async def test_get_solver_job_pricing_unit( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -169,7 +169,7 @@ def _get_pricing_unit_side_effect( return capture.response_body create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / capture_file, side_effects_callbacks=( [_get_job_side_effect, _get_pricing_unit_side_effect] @@ -202,8 +202,8 @@ def _get_pricing_unit_side_effect( ) async def test_start_solver_job_pricing_unit_with_payment( client: AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, mocked_groups_extra_properties, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, @@ -256,8 +256,8 @@ def _put_pricing_plan_and_unit_side_effect( _put_pricing_plan_and_unit_side_effect.was_called = False create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, ], capture_path=project_tests_dir / "mocks" / capture_name, side_effects_callbacks=callbacks, @@ -279,8 +279,8 @@ def _put_pricing_plan_and_unit_side_effect( async def test_get_solver_job_pricing_unit_no_payment( client: AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, mocked_groups_extra_properties, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, @@ -293,8 +293,8 @@ async def test_get_solver_job_pricing_unit_no_payment( create_respx_mock_from_capture( respx_mocks=[ - mocked_directorv2_service_api_base, - mocked_webserver_service_api_base, + mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "start_job_no_payment.json", side_effects_callbacks=[ @@ -314,8 +314,8 @@ async def test_get_solver_job_pricing_unit_no_payment( async def test_start_solver_job_conflict( client: AsyncClient, - mocked_webserver_service_api_base, - mocked_directorv2_service_api_base, + mocked_webserver_rest_api_base, + mocked_directorv2_rest_api_base, mocked_groups_extra_properties, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, @@ -328,8 +328,8 @@ async def test_start_solver_job_conflict( create_respx_mock_from_capture( respx_mocks=[ - mocked_directorv2_service_api_base, - mocked_webserver_service_api_base, + mocked_directorv2_rest_api_base, + mocked_webserver_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "start_solver_job.json", side_effects_callbacks=[ @@ -350,7 +350,7 @@ async def test_start_solver_job_conflict( async def test_stop_job( client: AsyncClient, - mocked_directorv2_service_api_base, + mocked_directorv2_rest_api_base, mocked_groups_extra_properties, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, @@ -372,7 +372,7 @@ def _stop_job_side_effect( return jsonable_encoder(task) create_respx_mock_from_capture( - respx_mocks=[mocked_directorv2_service_api_base], + respx_mocks=[mocked_directorv2_rest_api_base], capture_path=project_tests_dir / "mocks" / "stop_job.json", side_effects_callbacks=[ _stop_job_side_effect, @@ -396,8 +396,8 @@ def _stop_job_side_effect( ) async def test_get_solver_job_outputs( client: AsyncClient, - mocked_webserver_service_api_base, - mocked_storage_service_api_base, + mocked_webserver_rest_api_base, + mocked_storage_rest_api_base, mocked_groups_extra_properties, mocked_solver_job_outputs, create_respx_mock_from_capture: CreateRespxMockCallback, @@ -433,8 +433,8 @@ def _wallet_side_effect( create_respx_mock_from_capture( respx_mocks=[ - mocked_webserver_service_api_base, - mocked_storage_service_api_base, + mocked_webserver_rest_api_base, + mocked_storage_rest_api_base, ], capture_path=project_tests_dir / "mocks" / "get_solver_outputs.json", side_effects_callbacks=[_sf, _sf, _sf, _wallet_side_effect, _sf], diff --git a/services/api-server/tests/unit/test_api_solvers.py b/services/api-server/tests/unit/test_api_solvers.py index 31e8ccb7f595..d35b648629ea 100644 --- a/services/api-server/tests/unit/test_api_solvers.py +++ b/services/api-server/tests/unit/test_api_solvers.py @@ -26,7 +26,7 @@ ) async def test_get_solver_pricing_plan( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -35,7 +35,7 @@ async def test_get_solver_pricing_plan( ): respx_mock = create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / capture, side_effects_callbacks=[], ) diff --git a/services/api-server/tests/unit/test_api_wallets.py b/services/api-server/tests/unit/test_api_wallets.py index bf6f7167f238..5249d94565ad 100644 --- a/services/api-server/tests/unit/test_api_wallets.py +++ b/services/api-server/tests/unit/test_api_wallets.py @@ -25,7 +25,7 @@ ) async def test_get_wallet( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -45,7 +45,7 @@ def _get_wallet_side_effect( return response create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / capture, side_effects_callbacks=[_get_wallet_side_effect], ) @@ -65,14 +65,14 @@ def _get_wallet_side_effect( async def test_get_default_wallet( client: AsyncClient, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, ): create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / "get_default_wallet.json", side_effects_callbacks=[], ) diff --git a/services/api-server/tests/unit/test_credits.py b/services/api-server/tests/unit/test_credits.py index f9548949b818..c78165a78f0b 100644 --- a/services/api-server/tests/unit/test_credits.py +++ b/services/api-server/tests/unit/test_credits.py @@ -10,13 +10,13 @@ async def test_get_credits_price( client: AsyncClient, auth: BasicAuth, - mocked_webserver_service_api_base, + mocked_webserver_rest_api_base, create_respx_mock_from_capture: CreateRespxMockCallback, project_tests_dir: Path, ): create_respx_mock_from_capture( - respx_mocks=[mocked_webserver_service_api_base], + respx_mocks=[mocked_webserver_rest_api_base], capture_path=project_tests_dir / "mocks" / "get_credits_price.json", side_effects_callbacks=[], ) diff --git a/services/api-server/tests/unit/test_models_api_resources.py b/services/api-server/tests/unit/test_models_api_resources.py index 46f31b3ecd7b..39137bcb8d07 100644 --- a/services/api-server/tests/unit/test_models_api_resources.py +++ b/services/api-server/tests/unit/test_models_api_resources.py @@ -7,8 +7,11 @@ from simcore_service_api_server.models.api_resources import ( compose_resource_name, + parse_collections_ids, parse_last_resource_id, + parse_resources_ids, split_resource_name, + split_resource_name_as_dict, ) @@ -38,3 +41,19 @@ def test_parse_resource_id(): assert ( parse_last_resource_id(resource_name) == split_resource_name(resource_name)[-1] ) + + collection_to_resource_id_map = split_resource_name_as_dict(resource_name) + # Collection-ID -> Resource-ID + assert list(collection_to_resource_id_map.keys()) == parse_collections_ids( + resource_name + ) + assert list(collection_to_resource_id_map.values()) == parse_resources_ids( + resource_name + ) + + assert collection_to_resource_id_map["solvers"] == "simcore/services/comp/isolve" + assert collection_to_resource_id_map["releases"] == "1.3.4" + assert ( + collection_to_resource_id_map["jobs"] == "f622946d-fd29-35b9-a193-abdd1095167c" + ) + assert collection_to_resource_id_map["outputs"] == "output 22" diff --git a/services/api-server/tests/unit/test_services_directorv2.py b/services/api-server/tests/unit/test_services_directorv2.py index bd5e6d9a7ae6..e30187c17077 100644 --- a/services/api-server/tests/unit/test_services_directorv2.py +++ b/services/api-server/tests/unit/test_services_directorv2.py @@ -28,7 +28,7 @@ def api() -> DirectorV2Api: async def test_oec_139646582688800_missing_ctx_values_for_msg_template( - mocked_directorv2_service_api_base: MockRouter, + mocked_directorv2_rest_api_base: MockRouter, project_id: ProjectID, user_id: UserID, api: DirectorV2Api, @@ -44,7 +44,7 @@ async def test_oec_139646582688800_missing_ctx_values_for_msg_template( # httpx.HTTPStatusError: Client error '404 Not Found' for url '/v2/computations/c7ad07d3-513f-4368-bcf0-354143b6a048?user_id=94' for method in ("GET", "POST", "DELETE"): - mocked_directorv2_service_api_base.request( + mocked_directorv2_rest_api_base.request( method, path__regex=r"/v2/computations/", ).respond(status_code=status.HTTP_404_NOT_FOUND) diff --git a/services/api-server/tests/unit/test_services_rabbitmq.py b/services/api-server/tests/unit/test_services_rabbitmq.py index aa644e81500a..ab7652a82526 100644 --- a/services/api-server/tests/unit/test_services_rabbitmq.py +++ b/services/api-server/tests/unit/test_services_rabbitmq.py @@ -327,7 +327,7 @@ async def log_streamer_with_distributor( app: FastAPI, project_id: ProjectID, user_id: UserID, - mocked_directorv2_service_api_base: respx.MockRouter, + mocked_directorv2_rest_api_base: respx.MockRouter, computation_done: Callable[[], bool], log_distributor: LogDistributor, ) -> AsyncIterable[LogStreamer]: @@ -342,7 +342,7 @@ def _get_computation(request: httpx.Request, **kwargs) -> httpx.Response: status_code=status.HTTP_200_OK, json=jsonable_encoder(task) ) - mocked_directorv2_service_api_base.get(f"/v2/computations/{project_id}").mock( + mocked_directorv2_rest_api_base.get(f"/v2/computations/{project_id}").mock( side_effect=_get_computation ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest_schemas.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest_schemas.py index 55834b7b6585..08de1844520b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest_schemas.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest_schemas.py @@ -37,26 +37,30 @@ class ProjectCreateHeaders(BaseModel): - - simcore_user_agent: str = Field( # type: ignore[literal-required] - default=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, - description="Optional simcore user agent", - alias=X_SIMCORE_USER_AGENT, - ) - - parent_project_uuid: ProjectID | None = Field( # type: ignore[literal-required] - default=None, - description="Optional parent project UUID", - alias=X_SIMCORE_PARENT_PROJECT_UUID, - ) - parent_node_id: NodeID | None = Field( # type: ignore[literal-required] - default=None, - description="Optional parent node ID", - alias=X_SIMCORE_PARENT_NODE_ID, - ) + simcore_user_agent: Annotated[ + str, + Field( + description="Optional simcore user agent", + alias=X_SIMCORE_USER_AGENT, + ), + ] = UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + parent_project_uuid: Annotated[ + ProjectID | None, + Field( + description="Optional parent project UUID", + alias=X_SIMCORE_PARENT_PROJECT_UUID, + ), + ] = None + parent_node_id: Annotated[ + NodeID | None, + Field( + description="Optional parent node ID", + alias=X_SIMCORE_PARENT_NODE_ID, + ), + ] = None @model_validator(mode="after") - def check_parent_valid(self) -> Self: + def _check_parent_valid(self) -> Self: if (self.parent_project_uuid is None and self.parent_node_id is not None) or ( self.parent_project_uuid is not None and self.parent_node_id is None ): @@ -68,34 +72,52 @@ def check_parent_valid(self) -> Self: class ProjectCreateQueryParams(BaseModel): - from_study: ProjectID | None = Field( - None, - description="Option to create a project from existing template or study: from_study={study_uuid}", - ) - as_template: bool = Field( - default=False, - description="Option to create a template from existing project: as_template=true", - ) - copy_data: bool = Field( - default=True, - description="Option to copy data when creating from an existing template or as a template, defaults to True", - ) - hidden: bool = Field( - default=False, - description="Enables/disables hidden flag. Hidden projects are by default unlisted", - ) + from_study: Annotated[ + ProjectID | None, + Field( + description="Option to create a project from existing template or study: from_study={study_uuid}", + ), + ] = None + as_template: Annotated[ + bool, + Field( + description="Option to create a template from existing project: as_template=true", + ), + ] = False + copy_data: Annotated[ + bool, + Field( + description="Option to copy data when creating from an existing template or as a template, defaults to True", + ), + ] = True + hidden: Annotated[ + bool, + Field( + description="Enables/disables hidden flag. Hidden projects are by default unlisted", + ), + ] = False model_config = ConfigDict(extra="forbid") class ProjectFilters(Filters): - trashed: bool | None = Field( - default=False, - description="Set to true to list trashed, false to list non-trashed (default), None to list all", - ) - search_by_project_name: str | None = Field( - default=None, - description="A search query to filter projects by their name. This field performs a case-insensitive partial match against the project name field.", - ) + trashed: Annotated[ + bool | None, + Field( + description="Set to true to list trashed, false to list non-trashed (default), None to list all", + ), + ] = False + search_by_project_name: Annotated[ + str | None, + Field( + description="A search query to filter projects by their name. This field performs a case-insensitive partial match against the project name field.", + ), + ] = None + job_parent_resource_name: Annotated[ + str | None, + Field( + description="A search query to filter projects with associated job_parent_resource_name", + ), + ] = None ProjectsListOrderParams = create_ordering_query_model_class( @@ -113,28 +135,34 @@ class ProjectFilters(Filters): class ProjectsListExtraQueryParams(RequestParameters): - project_type: ProjectTypeAPI = Field(default=ProjectTypeAPI.all, alias="type") - show_hidden: bool = Field( - default=False, description="includes projects marked as hidden in the listing" - ) - search: str | None = Field( - default=None, - description="Multi column full text search", - max_length=100, - examples=["My Project"], - ) - folder_id: FolderID | None = Field( - default=None, - description="Filter projects in specific folder. Default filtering is a root directory.", - ) - workspace_id: WorkspaceID | None = Field( - default=None, - description="Filter projects in specific workspace. Default filtering is a private workspace.", - ) + project_type: Annotated[ProjectTypeAPI, Field(alias="type")] = ProjectTypeAPI.all + show_hidden: Annotated[ + bool, Field(description="includes projects marked as hidden in the listing") + ] = False + search: Annotated[ + str | None, + Field( + description="Multi column full text search", + max_length=100, + examples=["My Project"], + ), + ] = None + folder_id: Annotated[ + FolderID | None, + Field( + description="Filter projects in specific folder. Default filtering is a root directory.", + ), + ] = None + workspace_id: Annotated[ + WorkspaceID | None, + Field( + description="Filter projects in specific workspace. Default filtering is a private workspace.", + ), + ] = None @field_validator("search", mode="before") @classmethod - def search_check_empty_string(cls, v): + def _search_check_empty_string(cls, v): if not v: return None return v @@ -157,27 +185,28 @@ class ProjectsListQueryParams( class ProjectActiveQueryParams(BaseModel): - client_session_id: str + client_session_id: Annotated[str, Field()] class ProjectSearchExtraQueryParams( PageQueryParameters, FiltersQueryParameters[ProjectFilters], ): - text: str | None = Field( - default=None, - description="Multi column full text search, across all folders and workspaces", - max_length=100, - examples=["My Project"], - ) + text: Annotated[ + str | None, + Field( + description="Multi column full text search, across all folders and workspaces", + max_length=100, + examples=["My Project"], + ), + ] = None tag_ids: Annotated[ str | None, Field( - default=None, description="Search by tag ID (multiple tag IDs may be provided separated by column)", examples=["1,3"], ), - ] + ] = None _empty_is_none = field_validator("text", mode="before")( empty_str_to_none_pre_validator diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py new file mode 100644 index 000000000000..9c37f87ae567 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py @@ -0,0 +1,55 @@ +from aiohttp import web +from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE +from models_library.products import ProductName +from models_library.projects import ProjectID +from models_library.users import UserID +from pydantic import ValidationError, validate_call +from servicelib.rabbitmq import RPCRouter +from servicelib.rabbitmq.rpc_interfaces.webserver.errors import ( + ProjectForbiddenRpcError, + ProjectNotFoundRpcError, +) + +from ...rabbitmq import get_rabbitmq_rpc_server +from .. import _jobs_service +from ..exceptions import ProjectInvalidRightsError, ProjectNotFoundError + +router = RPCRouter() + + +@router.expose( + reraise_if_error_type=( + ProjectForbiddenRpcError, + ProjectNotFoundRpcError, + ValidationError, + ) +) +@validate_call(config={"arbitrary_types_allowed": True}) +async def mark_project_as_job( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + project_uuid: ProjectID, + job_parent_resource_name: str, +) -> None: + + try: + + await _jobs_service.set_project_as_job( + app, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name=job_parent_resource_name, + ) + except ProjectInvalidRightsError as err: + raise ProjectForbiddenRpcError.from_domain_error(err) from err + + except ProjectNotFoundError as err: + raise ProjectNotFoundRpcError.from_domain_error(err) from err + + +async def register_rpc_routes_on_startup(app: web.Application): + rpc_server = get_rabbitmq_rpc_server(app) + await rpc_server.register_router(router, WEBSERVER_RPC_NAMESPACE, app) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py new file mode 100644 index 000000000000..385f0fb9a2cf --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -0,0 +1,36 @@ +import logging + +from models_library.projects import ProjectID +from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs +from simcore_postgres_database.utils_repos import transaction_context +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.base_repository import BaseRepository + +_logger = logging.getLogger(__name__) + + +class ProjectJobsRepository(BaseRepository): + + async def set_project_as_job( + self, + connection: AsyncConnection | None = None, + *, + project_uuid: ProjectID, + job_parent_resource_name: str, + ) -> None: + async with transaction_context(self.engine, connection) as conn: + stmt = ( + pg_insert(projects_to_jobs) + .values( + project_uuid=f"{project_uuid}", + job_parent_resource_name=job_parent_resource_name, + ) + .on_conflict_do_update( + index_elements=["project_uuid", "job_parent_resource_name"], + set_={"job_parent_resource_name": job_parent_resource_name}, + ) + ) + + await conn.execute(stmt) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py new file mode 100644 index 000000000000..508a7eb232e4 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py @@ -0,0 +1,47 @@ +import logging +from typing import Annotated + +from aiohttp import web +from models_library.products import ProductName +from models_library.projects import ProjectID +from models_library.users import UserID +from pydantic import AfterValidator, validate_call + +from ._access_rights_service import check_user_project_permission +from ._jobs_repository import ProjectJobsRepository + +_logger = logging.getLogger(__name__) + + +def _validate_job_parent_resource_name(value: str) -> str: + if value and not value.startswith("/") and not value.endswith("/") and "/" in value: + return value + msg = "Invalid format: must contain '/' but cannot start or end with '/'" + raise ValueError(msg) + + +@validate_call(config={"arbitrary_types_allowed": True}) +async def set_project_as_job( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + project_uuid: ProjectID, + job_parent_resource_name: Annotated[ + str, AfterValidator(_validate_job_parent_resource_name) + ], +) -> None: + + await check_user_project_permission( + app, + project_id=project_uuid, + user_id=user_id, + product_name=product_name, + permission="write", + ) + + repo = ProjectJobsRepository.create_from_app(app) + + await repo.set_project_as_job( + project_uuid=project_uuid, job_parent_resource_name=job_parent_resource_name + ) diff --git a/services/web/server/src/simcore_service_webserver/projects/plugin.py b/services/web/server/src/simcore_service_webserver/projects/plugin.py index f968908c7973..0500af2c3dd2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/plugin.py +++ b/services/web/server/src/simcore_service_webserver/projects/plugin.py @@ -10,6 +10,7 @@ from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from ..constants import APP_SETTINGS_KEY +from ..rabbitmq import setup_rabbitmq from ._controller import ( comments_rest, folders_rest, @@ -19,13 +20,14 @@ nodes_rest, ports_rest, projects_rest, + projects_rpc, + projects_slot, projects_states_rest, tags_rest, trash_rest, wallets_rest, workspaces_rest, ) -from ._controller.projects_slot import setup_project_observer_events from ._projects_repository_legacy import setup_projects_db from ._security_service import setup_projects_access @@ -48,9 +50,15 @@ def setup_projects(app: web.Application) -> bool: # database API setup_projects_db(app) - # registers event handlers (e.g. on_user_disconnect) - setup_project_observer_events(app) + # setup SLOT-controllers + projects_slot.setup_project_observer_events(app) + # setup RPC-controllers + setup_rabbitmq(app) + if app[APP_SETTINGS_KEY].WEBSERVER_RABBITMQ: + app.on_startup.append(projects_rpc.register_rpc_routes_on_startup) + + # setup REST-controllers app.router.add_routes(projects_states_rest.routes) app.router.add_routes(projects_rest.routes) app.router.add_routes(comments_rest.routes) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py new file mode 100644 index 000000000000..3cc8dde0e0b0 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py @@ -0,0 +1,151 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + + +from collections.abc import AsyncIterator, Awaitable, Callable +from uuid import UUID + +import pytest +from aiohttp.test_utils import TestClient +from common_library.users_enums import UserRole +from models_library.products import ProductName +from models_library.projects import ProjectID +from pydantic import ValidationError +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from pytest_simcore.helpers.webserver_login import NewUser, UserInfoDict +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc +from servicelib.rabbitmq.rpc_interfaces.webserver.errors import ( + ProjectForbiddenRpcError, + ProjectNotFoundRpcError, +) +from settings_library.rabbit import RabbitSettings +from simcore_service_webserver.application_settings import ApplicationSettings +from simcore_service_webserver.projects.models import ProjectDict + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.fixture +def user_role() -> UserRole: + # for logged_user + return UserRole.USER + + +@pytest.fixture +def app_environment( + rabbit_service: RabbitSettings, + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, +): + new_envs = setenvs_from_dict( + monkeypatch, + { + **app_environment, + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + }, + ) + + settings = ApplicationSettings.create_from_envs() + assert settings.WEBSERVER_RABBITMQ + + return new_envs + + +@pytest.fixture +async def rpc_client( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], +) -> RabbitMQRPCClient: + return await rabbitmq_rpc_client("client") + + +async def test_rpc_client_mark_project_as_job( + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + logged_user: UserInfoDict, + user_project: ProjectDict, +): + # `logged_user` OWNS the `user_project` but not `other_user` + project_uuid: ProjectID = UUID(user_project["uuid"]) + user_id = logged_user["id"] + + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name="solvers/solver123/version/1.2.3", + ) + + +@pytest.fixture +async def other_user( + client: TestClient, + logged_user: UserInfoDict, +) -> AsyncIterator[UserInfoDict]: + + async with NewUser( + user_data={ + "name": "other-user", + "email": "other-user" + logged_user["email"], + }, + app=client.app, + ) as other_user_info: + + assert other_user_info["name"] != logged_user["name"] + yield other_user_info + + +async def test_errors_on_rpc_client_mark_project_as_job( + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + logged_user: UserInfoDict, + other_user: UserInfoDict, + user_project: ProjectDict, +): + # `logged_user` OWNS the `user_project` but not `other_user` + project_uuid: ProjectID = UUID(user_project["uuid"]) + user_id = logged_user["id"] + other_user_id = other_user["id"] + + with pytest.raises(ProjectForbiddenRpcError) as exc_info: + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=other_user_id, # <-- no access + project_uuid=project_uuid, + job_parent_resource_name="solvers/solver123/version/1.2.3", + ) + + assert exc_info.value.error_context()["project_uuid"] == project_uuid + + with pytest.raises(ProjectNotFoundRpcError, match="not found"): + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=logged_user["id"], + project_uuid=UUID("00000000-0000-0000-0000-000000000000"), # <-- wont find + job_parent_resource_name="solvers/solver123/version/1.2.3", + ) + + with pytest.raises(ValidationError, match="job_parent_resource_name") as exc_info: + await projects_rpc.mark_project_as_job( + rpc_client=rpc_client, + product_name=product_name, + user_id=user_id, + project_uuid=project_uuid, + job_parent_resource_name="This is not a resource", # <-- wrong format + ) + + assert exc_info.value.error_count() == 1 + assert exc_info.value.errors()[0]["loc"] == ("job_parent_resource_name",) + assert exc_info.value.errors()[0]["type"] == "value_error" diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py index 3498fd2abcb3..b2a885451602 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py @@ -7,7 +7,6 @@ import asyncio import re import urllib.parse -from collections.abc import AsyncIterator from typing import Any import pytest @@ -89,7 +88,7 @@ def web_server(redis_service: RedisSettings, web_server: TestServer) -> TestServ @pytest.fixture(autouse=True) async def director_v2_automock( director_v2_service_mock: aioresponses, -) -> AsyncIterator[aioresponses]: +) -> aioresponses: return director_v2_service_mock