From 1343987ff621db62c647bae4d8f92dad224b4c3e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 26 May 2025 13:26:22 +0200 Subject: [PATCH 1/5] ensure test run if tools are not installed --- .../simcore-sdk/tests/integration/conftest.py | 16 ++++++++-------- .../test_node_ports_common_filemanager.py | 16 +++++++++------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index b3aba7d8d356..b32fc4aa1dfa 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -337,7 +337,7 @@ def _assign_config( @pytest.fixture async def r_clone_settings_factory( minio_s3_settings: S3Settings, storage_service: URL -) -> Awaitable[RCloneSettings]: +) -> Callable[[], Awaitable[RCloneSettings]]: async def _factory() -> RCloneSettings: settings = RCloneSettings( R_CLONE_S3=minio_s3_settings, R_CLONE_PROVIDER=S3Provider.MINIO @@ -347,13 +347,13 @@ async def _factory() -> RCloneSettings: return settings - return _factory() + return _factory @pytest.fixture async def aws_s3_cli_settings_factory( minio_s3_settings: S3Settings, storage_service: URL -) -> Awaitable[AwsS3CliSettings]: +) -> Callable[[], Awaitable[AwsS3CliSettings]]: async def _factory() -> AwsS3CliSettings: settings = AwsS3CliSettings(AWS_S3_CLI_S3=minio_s3_settings) if not await is_aws_s3_cli_available(settings): @@ -361,21 +361,21 @@ async def _factory() -> AwsS3CliSettings: return settings - return _factory() + return _factory @pytest.fixture async def r_clone_settings( - r_clone_settings_factory: Awaitable[RCloneSettings], + r_clone_settings_factory: Callable[[], Awaitable[RCloneSettings]], ) -> RCloneSettings: - return await r_clone_settings_factory + return await r_clone_settings_factory() @pytest.fixture async def aws_s3_cli_settings( - aws_s3_cli_settings_factory: Awaitable[AwsS3CliSettings], + aws_s3_cli_settings_factory: Callable[[], Awaitable[AwsS3CliSettings]], ) -> AwsS3CliSettings: - return await aws_s3_cli_settings_factory + return await aws_s3_cli_settings_factory() @pytest.fixture diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py b/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py index da7bef85cbee..5867b6d16c9b 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py @@ -56,15 +56,17 @@ class _SyncSettings(BaseModel): "Both RClone and AwsS3Cli disabled", ], ) -def optional_sync_settings( - r_clone_settings: RCloneSettings, - aws_s3_cli_settings: AwsS3CliSettings, +async def optional_sync_settings( + r_clone_settings_factory: Callable[[], Awaitable[RCloneSettings]], + aws_s3_cli_settings_factory: Callable[[], Awaitable[AwsS3CliSettings]], request: pytest.FixtureRequest, ) -> _SyncSettings: _rclone_enabled, _aws_s3_cli_enabled = request.param - _r_clone_settings = r_clone_settings if _rclone_enabled else None - _aws_s3_cli_settings = aws_s3_cli_settings if _aws_s3_cli_enabled else None + _r_clone_settings = await r_clone_settings_factory() if _rclone_enabled else None + _aws_s3_cli_settings = ( + await aws_s3_cli_settings_factory() if _aws_s3_cli_enabled else None + ) return _SyncSettings( r_clone_settings=_r_clone_settings, aws_s3_cli_settings=_aws_s3_cli_settings @@ -82,8 +84,8 @@ def _file_size(size_str: str, **pytest_params): [ _file_size("10Mib"), _file_size("103Mib"), - _file_size("1003Mib", marks=pytest.mark.heavy_load), - _file_size("7Gib", marks=pytest.mark.heavy_load), + _file_size("1003Mib"), + _file_size("7Gib"), ], ids=byte_size_ids, ) From 3fa4beeb78e4a915c98288b83c85065ae25cf221 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 26 May 2025 13:28:15 +0200 Subject: [PATCH 2/5] revert --- .../tests/integration/test_node_ports_common_filemanager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py b/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py index 5867b6d16c9b..5f0656a8c59d 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py @@ -84,8 +84,8 @@ def _file_size(size_str: str, **pytest_params): [ _file_size("10Mib"), _file_size("103Mib"), - _file_size("1003Mib"), - _file_size("7Gib"), + _file_size("1003Mib", marks=pytest.mark.heavy_load), + _file_size("7Gib", marks=pytest.mark.heavy_load), ], ids=byte_size_ids, ) From 1d86fa5599fa9065031b32b1d1e8501690c88c19 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 26 May 2025 13:52:47 +0200 Subject: [PATCH 3/5] default block for computing checksums --- packages/service-library/src/servicelib/file_utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/service-library/src/servicelib/file_utils.py b/packages/service-library/src/servicelib/file_utils.py index 2d4f47428339..f41b210262a9 100644 --- a/packages/service-library/src/servicelib/file_utils.py +++ b/packages/service-library/src/servicelib/file_utils.py @@ -1,10 +1,11 @@ import asyncio import hashlib import shutil +from collections.abc import Iterator from contextlib import contextmanager from logging import Logger from pathlib import Path -from typing import Final, Iterator, Protocol +from typing import Final, Protocol # https://docs.python.org/3/library/shutil.html#shutil.rmtree # https://docs.python.org/3/library/os.html#os.remove @@ -13,11 +14,13 @@ from pydantic import ByteSize, TypeAdapter CHUNK_4KB: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("4kb") # 4K blocks +CHUNK_8MB: Final[ByteSize] = TypeAdapter(ByteSize).validate_python( + "8MiB" +) # 8mIB blocks class AsyncStream(Protocol): - async def read(self, size: int = -1) -> bytes: - ... + async def read(self, size: int = -1) -> bytes: ... _shutil_rmtree = sync_to_async(shutil.rmtree) @@ -45,7 +48,7 @@ async def remove_directory( async def create_sha256_checksum( - async_stream: AsyncStream, *, chunk_size: ByteSize = CHUNK_4KB + async_stream: AsyncStream, *, chunk_size: ByteSize = CHUNK_8MB ) -> str: """ Usage: From 735613671f01224887e7892ddbe2a92b7516ca5a Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 26 May 2025 13:52:55 +0200 Subject: [PATCH 4/5] ruff --- .../src/simcore_sdk/node_ports_common/filemanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py index b23f65b290cb..5fdd631474d1 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py @@ -291,7 +291,7 @@ async def _generate_checksum( return checksum if isinstance(path_to_upload, Path): async with aiofiles.open(path_to_upload, mode="rb") as f: - checksum = SHA256Str(await create_sha256_checksum(f)) + checksum = await create_sha256_checksum(f) elif isinstance(path_to_upload, UploadableFileObject): checksum = path_to_upload.sha256_checksum return checksum From 60704c6adb7bc1cae7b33392153f4d36083963bc Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 27 May 2025 08:23:45 +0200 Subject: [PATCH 5/5] fix test --- .../test_node_ports_v2_nodeports2.py | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py b/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py index f9f189d01c4a..88d16e383d25 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py @@ -21,7 +21,6 @@ import pytest import sqlalchemy as sa from faker import Faker -from models_library.projects import ProjectIDStr from models_library.projects_nodes_io import ( BaseFileLink, DownloadLink, @@ -156,10 +155,10 @@ def config_value_symlink_path(symlink_path: Path) -> dict[str, Any]: @pytest.fixture(params=[True, False]) async def option_r_clone_settings( - request, r_clone_settings_factory: Awaitable[RCloneSettings] + request, r_clone_settings_factory: Callable[[], Awaitable[RCloneSettings]] ) -> RCloneSettings | None: if request.param: - return await r_clone_settings_factory + return await r_clone_settings_factory() return None @@ -174,7 +173,7 @@ async def test_default_configuration( await check_config_valid( await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ), @@ -192,7 +191,7 @@ async def test_invalid_ports( config_dict, _, _ = create_special_configuration() PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -238,7 +237,7 @@ async def test_port_value_accessors( PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -298,7 +297,7 @@ async def test_port_file_accessors( PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -375,7 +374,7 @@ async def test_adding_new_ports( config_dict, project_id, node_uuid = create_special_configuration() PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -429,7 +428,7 @@ async def test_removing_ports( ) # pylint: disable=W0612 PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -489,7 +488,7 @@ async def test_get_value_from_previous_node( PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -541,7 +540,7 @@ async def test_get_file_from_previous_node( ) PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -598,7 +597,7 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name( ) PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -659,7 +658,7 @@ async def test_file_mapping( ) PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -752,7 +751,7 @@ async def test_regression_concurrent_port_update_fails( PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, ) @@ -841,7 +840,7 @@ async def test_batch_update_inputs_outputs( PORTS = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, )