From 4ff1761f97a540fc4a5689ae181bc217aadf6645 Mon Sep 17 00:00:00 2001 From: Camille Boillet Date: Thu, 29 Sep 2022 15:42:21 +0200 Subject: [PATCH 1/4] feat: archive datamanager fix migration fix migration post rebase Signed-off-by: Camille Boillet fix migrations post rebase Signed-off-by: Camille Boillet black Signed-off-by: Camille Boillet isort Signed-off-by: Camille Boillet lint Signed-off-by: Camille Boillet --- backend/api/events/sync.py | 1 + .../0040_add_archived_datamanager.py | 19 ++++++ ...puteplan_failed_task_category_and_more.py} | 2 +- backend/api/models/datamanager.py | 1 + .../api/tests/views/test_views_datamanager.py | 22 +++++++ backend/api/views/datamanager.py | 22 ++++++- backend/api/views/filters_utils.py | 14 +++++ backend/orchestrator/client.py | 7 +++ backend/orchestrator/datamanager_pb2.py | 60 ++++++++++++------- backend/orchestrator/datamanager_pb2.pyi | 31 +++++++++- backend/orchestrator/datamanager_pb2_grpc.py | 33 ++++++++++ backend/orchestrator/datamanager_pb2_grpc.pyi | 10 ++++ backend/orchestrator/mock.py | 15 +++++ backend/orchestrator/resources.py | 3 +- backend/substrapp/compute_tasks/context.py | 8 ++- .../substrapp/tests/compute_tasks/conftest.py | 21 +++++++ .../tests/compute_tasks/test_asset_buffer.py | 4 +- .../tests/compute_tasks/test_context.py | 9 +++ 18 files changed, 255 insertions(+), 27 deletions(-) create mode 100644 backend/api/migrations/0040_add_archived_datamanager.py rename backend/api/migrations/{0040_alter_computeplan_failed_task_category_and_more.py => 0041_alter_computeplan_failed_task_category_and_more.py} (96%) diff --git a/backend/api/events/sync.py b/backend/api/events/sync.py index 68567f438..a4d155834 100644 --- a/backend/api/events/sync.py +++ b/backend/api/events/sync.py @@ -273,6 +273,7 @@ def _update_datamanager(key: str, data: dict) -> None: datamanager = DataManager.objects.get(key=key) datamanager.name = data["name"] + datamanager.archived = data["archived"] datamanager.save() diff --git a/backend/api/migrations/0040_add_archived_datamanager.py b/backend/api/migrations/0040_add_archived_datamanager.py new file mode 100644 index 000000000..cfaf2d181 --- /dev/null +++ b/backend/api/migrations/0040_add_archived_datamanager.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0.7 on 2022-09-27 23:14 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0039_computetaskinputasset"), + ] + + operations = [ + migrations.AddField( + model_name="datamanager", + name="archived", + field=models.BooleanField(default=False), + ), + ] diff --git a/backend/api/migrations/0040_alter_computeplan_failed_task_category_and_more.py b/backend/api/migrations/0041_alter_computeplan_failed_task_category_and_more.py similarity index 96% rename from backend/api/migrations/0040_alter_computeplan_failed_task_category_and_more.py rename to backend/api/migrations/0041_alter_computeplan_failed_task_category_and_more.py index 69dc07def..41fcfd097 100644 --- a/backend/api/migrations/0040_alter_computeplan_failed_task_category_and_more.py +++ b/backend/api/migrations/0041_alter_computeplan_failed_task_category_and_more.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0039_computetaskinputasset"), + ("api", "0040_add_archived_datamanager"), ] operations = [ diff --git a/backend/api/models/datamanager.py b/backend/api/models/datamanager.py index 892a56155..b4a0d642a 100644 --- a/backend/api/models/datamanager.py +++ b/backend/api/models/datamanager.py @@ -25,6 +25,7 @@ class DataManager(models.Model, AssetPermissionMixin): creation_date = models.DateTimeField() metadata = models.JSONField() channel = models.CharField(max_length=100) + archived = models.BooleanField(default=False) def get_train_data_samples(self): # default ordering is based on an association table `TaskDataSamples` diff --git a/backend/api/tests/views/test_views_datamanager.py b/backend/api/tests/views/test_views_datamanager.py index 64838135f..e8b463305 100644 --- a/backend/api/tests/views/test_views_datamanager.py +++ b/backend/api/tests/views/test_views_datamanager.py @@ -180,6 +180,28 @@ def test_datamanager_update(self): response = self.client.put(url, data=data, format="json", **self.extra) self.assertEqual(response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR) + def test_datamanager_archive(self): + data_manager = self.expected_results[0] + data = { + "key": data_manager["key"], + "archived": True, + } + + url = reverse("api:data_manager-archive", args=[data_manager["key"]]) + data_manager["archived"] = data["archived"] + + with mock.patch.object(OrchestratorClient, "archive_datamanager", side_effect=data_manager): + response = self.client.put(url, data=data, format="json", **self.extra) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + error = OrcError() + error.code = StatusCode.INTERNAL + + with mock.patch.object(OrchestratorClient, "archive_datamanager", side_effect=error): + response = self.client.put(url, data=data, format="json", **self.extra) + self.assertEqual(response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR) + def test_datamanager_list_empty(self): DataManager.objects.all().delete() response = self.client.get(self.url, **self.extra) diff --git a/backend/api/views/datamanager.py b/backend/api/views/datamanager.py index 20ea683a0..4b3118808 100644 --- a/backend/api/views/datamanager.py +++ b/backend/api/views/datamanager.py @@ -26,6 +26,7 @@ from api.views.utils import ValidationExceptionError from api.views.utils import get_channel_name from api.views.utils import validate_metadata +from backend.settings.common import to_bool from libs.pagination import DefaultPageNumberPagination from substrapp.models import DataManager as DataManagerFiles from substrapp.orchestrator import get_orchestrator_client @@ -191,7 +192,7 @@ def update(self, request, *args, **kwargs): datamanager = self.get_object() name = request.data.get("name") - orc_algo = { + orc_datamanager = { "key": str(datamanager.key), "name": name, } @@ -199,7 +200,24 @@ def update(self, request, *args, **kwargs): # send update to orchestrator # the modification in local db will be done upon corresponding event reception with get_orchestrator_client(get_channel_name(request)) as client: - client.update_datamanager(orc_algo) + client.update_datamanager(orc_datamanager) + + return ApiResponse({}, status=status.HTTP_200_OK) + + @action(detail=True, methods=["PUT"]) + def archive(self, request, *args, **kwargs): + datamanager = self.get_object() + archived = request.data.get("archived") + + orc_algo = { + "key": str(datamanager.key), + "archived": to_bool(archived), + } + + # send archiving message to orchestrator + # the modification in local db will be done upon corresponding event reception + with get_orchestrator_client(get_channel_name(request)) as client: + client.archive_datamanager(orc_algo) return ApiResponse({}, status=status.HTTP_200_OK) diff --git a/backend/api/views/filters_utils.py b/backend/api/views/filters_utils.py index a6318306a..bcdd26461 100644 --- a/backend/api/views/filters_utils.py +++ b/backend/api/views/filters_utils.py @@ -13,6 +13,8 @@ from rest_framework.filters import BaseFilterBackend from rest_framework.filters import SearchFilter +from backend.settings.common import to_bool + logger = structlog.get_logger(__name__) @@ -141,3 +143,15 @@ def filter_queryset(self, request, queryset, view): django_filters[f'metadata_filters__{f["key"]}__icontains'] = f["value"] return queryset.filter(**django_filters) + + +class ArchiveFilter(BaseFilterBackend): + def filter_queryset(self, request, queryset, view): + # apply filters to queryset + if view.action == "list": + archived = to_bool(request.query_params.get("archived")) + if archived: + return queryset + return queryset.filter(archived=False) + else: + return queryset diff --git a/backend/orchestrator/client.py b/backend/orchestrator/client.py index 864e98a13..80da8da77 100644 --- a/backend/orchestrator/client.py +++ b/backend/orchestrator/client.py @@ -256,6 +256,13 @@ def update_datamanager(self, args): ) return MessageToDict(data, **CONVERT_SETTINGS) + @grpc_retry + def archive_datamanager(self, args): + data = self._datamanager_client.ArchiveDataManager( + datamanager_pb2.ArchiveDataManagerParam(**args), metadata=self._metadata + ) + return MessageToDict(data, **CONVERT_SETTINGS) + def _get_task_input(self, input: dict) -> computetask_pb2.ComputeTaskInput: """Convert a dict into a computetask_pb2.ComputeTaskInput""" diff --git a/backend/orchestrator/datamanager_pb2.py b/backend/orchestrator/datamanager_pb2.py index 41b11106e..c4e96b952 100644 --- a/backend/orchestrator/datamanager_pb2.py +++ b/backend/orchestrator/datamanager_pb2.py @@ -16,7 +16,7 @@ from . import common_pb2 as common__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x11\x64\x61tamanager.proto\x12\x0corchestrator\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x0c\x63ommon.proto\"\xa2\x03\n\x0b\x44\x61taManager\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05owner\x18\x03 \x01(\t\x12.\n\x0bpermissions\x18\x04 \x01(\x0b\x32\x19.orchestrator.Permissions\x12.\n\x0b\x64\x65scription\x18\x06 \x01(\x0b\x32\x19.orchestrator.Addressable\x12)\n\x06opener\x18\x07 \x01(\x0b\x32\x19.orchestrator.Addressable\x12\x0c\n\x04type\x18\x08 \x01(\t\x12\x31\n\rcreation_date\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x31\n\x0flogs_permission\x18\n \x01(\x0b\x32\x18.orchestrator.Permission\x12\x39\n\x08metadata\x18\x10 \x03(\x0b\x32\'.orchestrator.DataManager.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf1\x02\n\x0eNewDataManager\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x35\n\x0fnew_permissions\x18\x03 \x01(\x0b\x32\x1c.orchestrator.NewPermissions\x12.\n\x0b\x64\x65scription\x18\x05 \x01(\x0b\x32\x19.orchestrator.Addressable\x12)\n\x06opener\x18\x06 \x01(\x0b\x32\x19.orchestrator.Addressable\x12\x0c\n\x04type\x18\x07 \x01(\t\x12\x35\n\x0flogs_permission\x18\x08 \x01(\x0b\x32\x1c.orchestrator.NewPermissions\x12<\n\x08metadata\x18\x10 \x03(\x0b\x32*.orchestrator.NewDataManager.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\"\n\x13GetDataManagerParam\x12\x0b\n\x03key\x18\x01 \x01(\t\"?\n\x16QueryDataManagersParam\x12\x12\n\npage_token\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\r\"f\n\x19QueryDataManagersResponse\x12\x30\n\rdata_managers\x18\x01 \x03(\x0b\x32\x19.orchestrator.DataManager\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"3\n\x16UpdateDataManagerParam\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"\x1b\n\x19UpdateDataManagerResponse2\xfc\x02\n\x12\x44\x61taManagerService\x12N\n\x13RegisterDataManager\x12\x1c.orchestrator.NewDataManager\x1a\x19.orchestrator.DataManager\x12N\n\x0eGetDataManager\x12!.orchestrator.GetDataManagerParam\x1a\x19.orchestrator.DataManager\x12\x62\n\x11QueryDataManagers\x12$.orchestrator.QueryDataManagersParam\x1a\'.orchestrator.QueryDataManagersResponse\x12\x62\n\x11UpdateDataManager\x12$.orchestrator.UpdateDataManagerParam\x1a\'.orchestrator.UpdateDataManagerResponseB+Z)github.com/substra/orchestrator/lib/assetb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x11\x64\x61tamanager.proto\x12\x0corchestrator\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x0c\x63ommon.proto\"\xb4\x03\n\x0b\x44\x61taManager\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\r\n\x05owner\x18\x03 \x01(\t\x12.\n\x0bpermissions\x18\x04 \x01(\x0b\x32\x19.orchestrator.Permissions\x12.\n\x0b\x64\x65scription\x18\x06 \x01(\x0b\x32\x19.orchestrator.Addressable\x12)\n\x06opener\x18\x07 \x01(\x0b\x32\x19.orchestrator.Addressable\x12\x0c\n\x04type\x18\x08 \x01(\t\x12\x31\n\rcreation_date\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x31\n\x0flogs_permission\x18\n \x01(\x0b\x32\x18.orchestrator.Permission\x12\x39\n\x08metadata\x18\x10 \x03(\x0b\x32\'.orchestrator.DataManager.MetadataEntry\x12\x10\n\x08\x61rchived\x18\x11 \x01(\x08\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf1\x02\n\x0eNewDataManager\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x35\n\x0fnew_permissions\x18\x03 \x01(\x0b\x32\x1c.orchestrator.NewPermissions\x12.\n\x0b\x64\x65scription\x18\x05 \x01(\x0b\x32\x19.orchestrator.Addressable\x12)\n\x06opener\x18\x06 \x01(\x0b\x32\x19.orchestrator.Addressable\x12\x0c\n\x04type\x18\x07 \x01(\t\x12\x35\n\x0flogs_permission\x18\x08 \x01(\x0b\x32\x1c.orchestrator.NewPermissions\x12<\n\x08metadata\x18\x10 \x03(\x0b\x32*.orchestrator.NewDataManager.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\"\n\x13GetDataManagerParam\x12\x0b\n\x03key\x18\x01 \x01(\t\"?\n\x16QueryDataManagersParam\x12\x12\n\npage_token\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\r\"f\n\x19QueryDataManagersResponse\x12\x30\n\rdata_managers\x18\x01 \x03(\x0b\x32\x19.orchestrator.DataManager\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"3\n\x16UpdateDataManagerParam\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"8\n\x17\x41rchiveDataManagerParam\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x10\n\x08\x61rchived\x18\x02 \x01(\x08\"\x1b\n\x19UpdateDataManagerResponse\"\x1c\n\x1a\x41rchiveDataManagerResponse2\xe3\x03\n\x12\x44\x61taManagerService\x12N\n\x13RegisterDataManager\x12\x1c.orchestrator.NewDataManager\x1a\x19.orchestrator.DataManager\x12N\n\x0eGetDataManager\x12!.orchestrator.GetDataManagerParam\x1a\x19.orchestrator.DataManager\x12\x62\n\x11QueryDataManagers\x12$.orchestrator.QueryDataManagersParam\x1a\'.orchestrator.QueryDataManagersResponse\x12\x62\n\x11UpdateDataManager\x12$.orchestrator.UpdateDataManagerParam\x1a\'.orchestrator.UpdateDataManagerResponse\x12\x65\n\x12\x41rchiveDataManager\x12%.orchestrator.ArchiveDataManagerParam\x1a(.orchestrator.ArchiveDataManagerResponseB+Z)github.com/substra/orchestrator/lib/assetb\x06proto3') @@ -28,7 +28,9 @@ _QUERYDATAMANAGERSPARAM = DESCRIPTOR.message_types_by_name['QueryDataManagersParam'] _QUERYDATAMANAGERSRESPONSE = DESCRIPTOR.message_types_by_name['QueryDataManagersResponse'] _UPDATEDATAMANAGERPARAM = DESCRIPTOR.message_types_by_name['UpdateDataManagerParam'] +_ARCHIVEDATAMANAGERPARAM = DESCRIPTOR.message_types_by_name['ArchiveDataManagerParam'] _UPDATEDATAMANAGERRESPONSE = DESCRIPTOR.message_types_by_name['UpdateDataManagerResponse'] +_ARCHIVEDATAMANAGERRESPONSE = DESCRIPTOR.message_types_by_name['ArchiveDataManagerResponse'] DataManager = _reflection.GeneratedProtocolMessageType('DataManager', (_message.Message,), { 'MetadataEntry' : _reflection.GeneratedProtocolMessageType('MetadataEntry', (_message.Message,), { @@ -87,6 +89,13 @@ }) _sym_db.RegisterMessage(UpdateDataManagerParam) +ArchiveDataManagerParam = _reflection.GeneratedProtocolMessageType('ArchiveDataManagerParam', (_message.Message,), { + 'DESCRIPTOR' : _ARCHIVEDATAMANAGERPARAM, + '__module__' : 'datamanager_pb2' + # @@protoc_insertion_point(class_scope:orchestrator.ArchiveDataManagerParam) + }) +_sym_db.RegisterMessage(ArchiveDataManagerParam) + UpdateDataManagerResponse = _reflection.GeneratedProtocolMessageType('UpdateDataManagerResponse', (_message.Message,), { 'DESCRIPTOR' : _UPDATEDATAMANAGERRESPONSE, '__module__' : 'datamanager_pb2' @@ -94,6 +103,13 @@ }) _sym_db.RegisterMessage(UpdateDataManagerResponse) +ArchiveDataManagerResponse = _reflection.GeneratedProtocolMessageType('ArchiveDataManagerResponse', (_message.Message,), { + 'DESCRIPTOR' : _ARCHIVEDATAMANAGERRESPONSE, + '__module__' : 'datamanager_pb2' + # @@protoc_insertion_point(class_scope:orchestrator.ArchiveDataManagerResponse) + }) +_sym_db.RegisterMessage(ArchiveDataManagerResponse) + _DATAMANAGERSERVICE = DESCRIPTOR.services_by_name['DataManagerService'] if _descriptor._USE_C_DESCRIPTORS == False: @@ -104,23 +120,27 @@ _NEWDATAMANAGER_METADATAENTRY._options = None _NEWDATAMANAGER_METADATAENTRY._serialized_options = b'8\001' _DATAMANAGER._serialized_start=83 - _DATAMANAGER._serialized_end=501 - _DATAMANAGER_METADATAENTRY._serialized_start=454 - _DATAMANAGER_METADATAENTRY._serialized_end=501 - _NEWDATAMANAGER._serialized_start=504 - _NEWDATAMANAGER._serialized_end=873 - _NEWDATAMANAGER_METADATAENTRY._serialized_start=454 - _NEWDATAMANAGER_METADATAENTRY._serialized_end=501 - _GETDATAMANAGERPARAM._serialized_start=875 - _GETDATAMANAGERPARAM._serialized_end=909 - _QUERYDATAMANAGERSPARAM._serialized_start=911 - _QUERYDATAMANAGERSPARAM._serialized_end=974 - _QUERYDATAMANAGERSRESPONSE._serialized_start=976 - _QUERYDATAMANAGERSRESPONSE._serialized_end=1078 - _UPDATEDATAMANAGERPARAM._serialized_start=1080 - _UPDATEDATAMANAGERPARAM._serialized_end=1131 - _UPDATEDATAMANAGERRESPONSE._serialized_start=1133 - _UPDATEDATAMANAGERRESPONSE._serialized_end=1160 - _DATAMANAGERSERVICE._serialized_start=1163 - _DATAMANAGERSERVICE._serialized_end=1543 + _DATAMANAGER._serialized_end=519 + _DATAMANAGER_METADATAENTRY._serialized_start=472 + _DATAMANAGER_METADATAENTRY._serialized_end=519 + _NEWDATAMANAGER._serialized_start=522 + _NEWDATAMANAGER._serialized_end=891 + _NEWDATAMANAGER_METADATAENTRY._serialized_start=472 + _NEWDATAMANAGER_METADATAENTRY._serialized_end=519 + _GETDATAMANAGERPARAM._serialized_start=893 + _GETDATAMANAGERPARAM._serialized_end=927 + _QUERYDATAMANAGERSPARAM._serialized_start=929 + _QUERYDATAMANAGERSPARAM._serialized_end=992 + _QUERYDATAMANAGERSRESPONSE._serialized_start=994 + _QUERYDATAMANAGERSRESPONSE._serialized_end=1096 + _UPDATEDATAMANAGERPARAM._serialized_start=1098 + _UPDATEDATAMANAGERPARAM._serialized_end=1149 + _ARCHIVEDATAMANAGERPARAM._serialized_start=1151 + _ARCHIVEDATAMANAGERPARAM._serialized_end=1207 + _UPDATEDATAMANAGERRESPONSE._serialized_start=1209 + _UPDATEDATAMANAGERRESPONSE._serialized_end=1236 + _ARCHIVEDATAMANAGERRESPONSE._serialized_start=1238 + _ARCHIVEDATAMANAGERRESPONSE._serialized_end=1266 + _DATAMANAGERSERVICE._serialized_start=1269 + _DATAMANAGERSERVICE._serialized_end=1752 # @@protoc_insertion_point(module_scope) diff --git a/backend/orchestrator/datamanager_pb2.pyi b/backend/orchestrator/datamanager_pb2.pyi index 85bde6fa8..123fa3e7a 100644 --- a/backend/orchestrator/datamanager_pb2.pyi +++ b/backend/orchestrator/datamanager_pb2.pyi @@ -46,6 +46,7 @@ class DataManager(google.protobuf.message.Message): CREATION_DATE_FIELD_NUMBER: builtins.int LOGS_PERMISSION_FIELD_NUMBER: builtins.int METADATA_FIELD_NUMBER: builtins.int + ARCHIVED_FIELD_NUMBER: builtins.int key: builtins.str name: builtins.str owner: builtins.str @@ -62,6 +63,7 @@ class DataManager(google.protobuf.message.Message): def logs_permission(self) -> common_pb2.Permission: ... @property def metadata(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + archived: builtins.bool def __init__( self, *, @@ -75,9 +77,10 @@ class DataManager(google.protobuf.message.Message): creation_date: google.protobuf.timestamp_pb2.Timestamp | None = ..., logs_permission: common_pb2.Permission | None = ..., metadata: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + archived: builtins.bool = ..., ) -> None: ... def HasField(self, field_name: typing_extensions.Literal["creation_date", b"creation_date", "description", b"description", "logs_permission", b"logs_permission", "opener", b"opener", "permissions", b"permissions"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["creation_date", b"creation_date", "description", b"description", "key", b"key", "logs_permission", b"logs_permission", "metadata", b"metadata", "name", b"name", "opener", b"opener", "owner", b"owner", "permissions", b"permissions", "type", b"type"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["archived", b"archived", "creation_date", b"creation_date", "description", b"description", "key", b"key", "logs_permission", b"logs_permission", "metadata", b"metadata", "name", b"name", "opener", b"opener", "owner", b"owner", "permissions", b"permissions", "type", b"type"]) -> None: ... global___DataManager = DataManager @@ -203,6 +206,23 @@ class UpdateDataManagerParam(google.protobuf.message.Message): global___UpdateDataManagerParam = UpdateDataManagerParam +class ArchiveDataManagerParam(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + ARCHIVED_FIELD_NUMBER: builtins.int + key: builtins.str + archived: builtins.bool + def __init__( + self, + *, + key: builtins.str = ..., + archived: builtins.bool = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["archived", b"archived", "key", b"key"]) -> None: ... + +global___ArchiveDataManagerParam = ArchiveDataManagerParam + class UpdateDataManagerResponse(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -211,3 +231,12 @@ class UpdateDataManagerResponse(google.protobuf.message.Message): ) -> None: ... global___UpdateDataManagerResponse = UpdateDataManagerResponse + +class ArchiveDataManagerResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___ArchiveDataManagerResponse = ArchiveDataManagerResponse diff --git a/backend/orchestrator/datamanager_pb2_grpc.py b/backend/orchestrator/datamanager_pb2_grpc.py index 2850a803e..c1d29f2f1 100644 --- a/backend/orchestrator/datamanager_pb2_grpc.py +++ b/backend/orchestrator/datamanager_pb2_grpc.py @@ -34,6 +34,11 @@ def __init__(self, channel): request_serializer=datamanager__pb2.UpdateDataManagerParam.SerializeToString, response_deserializer=datamanager__pb2.UpdateDataManagerResponse.FromString, ) + self.ArchiveDataManager = channel.unary_unary( + '/orchestrator.DataManagerService/ArchiveDataManager', + request_serializer=datamanager__pb2.ArchiveDataManagerParam.SerializeToString, + response_deserializer=datamanager__pb2.ArchiveDataManagerResponse.FromString, + ) class DataManagerServiceServicer(object): @@ -63,6 +68,12 @@ def UpdateDataManager(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ArchiveDataManager(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_DataManagerServiceServicer_to_server(servicer, server): rpc_method_handlers = { @@ -86,6 +97,11 @@ def add_DataManagerServiceServicer_to_server(servicer, server): request_deserializer=datamanager__pb2.UpdateDataManagerParam.FromString, response_serializer=datamanager__pb2.UpdateDataManagerResponse.SerializeToString, ), + 'ArchiveDataManager': grpc.unary_unary_rpc_method_handler( + servicer.ArchiveDataManager, + request_deserializer=datamanager__pb2.ArchiveDataManagerParam.FromString, + response_serializer=datamanager__pb2.ArchiveDataManagerResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'orchestrator.DataManagerService', rpc_method_handlers) @@ -163,3 +179,20 @@ def UpdateDataManager(request, datamanager__pb2.UpdateDataManagerResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArchiveDataManager(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/orchestrator.DataManagerService/ArchiveDataManager', + datamanager__pb2.ArchiveDataManagerParam.SerializeToString, + datamanager__pb2.ArchiveDataManagerResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/backend/orchestrator/datamanager_pb2_grpc.pyi b/backend/orchestrator/datamanager_pb2_grpc.pyi index a1405a0e8..a03997852 100644 --- a/backend/orchestrator/datamanager_pb2_grpc.pyi +++ b/backend/orchestrator/datamanager_pb2_grpc.pyi @@ -24,6 +24,10 @@ class DataManagerServiceStub: datamanager_pb2.UpdateDataManagerParam, datamanager_pb2.UpdateDataManagerResponse, ] + ArchiveDataManager: grpc.UnaryUnaryMultiCallable[ + datamanager_pb2.ArchiveDataManagerParam, + datamanager_pb2.ArchiveDataManagerResponse, + ] class DataManagerServiceServicer(metaclass=abc.ABCMeta): @abc.abstractmethod @@ -50,5 +54,11 @@ class DataManagerServiceServicer(metaclass=abc.ABCMeta): request: datamanager_pb2.UpdateDataManagerParam, context: grpc.ServicerContext, ) -> datamanager_pb2.UpdateDataManagerResponse: ... + @abc.abstractmethod + def ArchiveDataManager( + self, + request: datamanager_pb2.ArchiveDataManagerParam, + context: grpc.ServicerContext, + ) -> datamanager_pb2.ArchiveDataManagerResponse: ... def add_DataManagerServiceServicer_to_server(servicer: DataManagerServiceServicer, server: grpc.Server) -> None: ... diff --git a/backend/orchestrator/mock.py b/backend/orchestrator/mock.py index 3599a6b2d..c0abc2598 100644 --- a/backend/orchestrator/mock.py +++ b/backend/orchestrator/mock.py @@ -1,5 +1,7 @@ import factory +from substrapp.tests.common import InputIdentifiers + from .resources import Address from .resources import Algo from .resources import AlgoInput @@ -8,6 +10,7 @@ from .resources import ComputeTask from .resources import ComputeTaskCategory from .resources import ComputeTaskInput +from .resources import ComputeTaskInputAsset from .resources import ComputeTaskOutput from .resources import ComputeTaskStatus from .resources import DataManager @@ -27,6 +30,17 @@ class Meta: parent_task_output_identifier = None +class ComputeTaskInputAssetFactory(factory.Factory): + class Meta: + model = ComputeTaskInputAsset + + identifier = InputIdentifiers.LOCAL + kind = AssetKind.ASSET_UNKNOWN + model = None + data_manager = None + data_sample = None + + class ComputeTaskFactory(factory.Factory): class Meta: model = ComputeTask @@ -76,6 +90,7 @@ class Meta: key = factory.Faker("uuid4") opener = factory.SubFactory(AddressFactory) + archived = False class AlgoInputFactory(factory.Factory): diff --git a/backend/orchestrator/resources.py b/backend/orchestrator/resources.py index 2d3a01d50..d88ce9fe8 100644 --- a/backend/orchestrator/resources.py +++ b/backend/orchestrator/resources.py @@ -112,11 +112,12 @@ class DataManager(pydantic.BaseModel): key: str opener: Address + archived: bool @classmethod def from_grpc(cls, m: datamanager_pb2.DataManager) -> DataManager: """Creates a DataManager from grpc message""" - return cls(key=m.key, opener=Address.from_grpc(m.opener)) + return cls(key=m.key, opener=Address.from_grpc(m.opener), archived=m.archived) class AlgoInput(pydantic.BaseModel): diff --git a/backend/substrapp/compute_tasks/context.py b/backend/substrapp/compute_tasks/context.py index f557e3b7f..e6e42fcc9 100644 --- a/backend/substrapp/compute_tasks/context.py +++ b/backend/substrapp/compute_tasks/context.py @@ -134,7 +134,13 @@ def data_manager(self) -> Optional[orchestrator.DataManager]: ] if len(dm) > 1: raise InvalidContextError("there are too many datamanagers") - return dm[0] if dm else None + + if not dm: + return None + + if dm[0].archived: + raise InvalidContextError(f"datamanager {dm[0].key} is archived and cannot be used for the task") + return dm[0] @property def data_sample_keys(self) -> list[str]: diff --git a/backend/substrapp/tests/compute_tasks/conftest.py b/backend/substrapp/tests/compute_tasks/conftest.py index 28986238b..1eb428315 100644 --- a/backend/substrapp/tests/compute_tasks/conftest.py +++ b/backend/substrapp/tests/compute_tasks/conftest.py @@ -5,8 +5,10 @@ import orchestrator import orchestrator.mock as orc_mock +from orchestrator.resources import AssetKind from substrapp.compute_tasks.context import Context from substrapp.compute_tasks.directories import Directories +from substrapp.tests.common import InputIdentifiers DOCKERFILE = """ FROM ubuntu:16.04 @@ -50,3 +52,22 @@ def testtuple_context(orc_metric) -> Context: directories=Directories(cp_key), has_chainkeys=False, ) + + +@pytest.fixture +def archived_datamanager_task_input_context(): + cp_key = str(uuid.uuid4()) + archived_dm = orc_mock.DataManagerFactory(archived=True) + task_input = orc_mock.ComputeTaskInputAsset( + identifier=InputIdentifiers.OPENER, kind=AssetKind.ASSET_DATA_MANAGER, data_manager=archived_dm + ) + + return Context( + channel_name="mychannel", + task=orc_mock.ComputeTaskFactory(), + compute_plan={}, + input_assets=[task_input], + algo=orc_mock.AlgoFactory(), + directories=Directories(cp_key), + has_chainkeys=False, + ) diff --git a/backend/substrapp/tests/compute_tasks/test_asset_buffer.py b/backend/substrapp/tests/compute_tasks/test_asset_buffer.py index a786acba7..ecf1e4562 100644 --- a/backend/substrapp/tests/compute_tasks/test_asset_buffer.py +++ b/backend/substrapp/tests/compute_tasks/test_asset_buffer.py @@ -159,7 +159,9 @@ def _setup_opener(self): self.opener_storage_address = "some storage address" self.data_manager_key = "some_data_manager_key" self.data_manager = DataManager( - key=self.data_manager_key, opener=Address(uri=self.opener_storage_address, checksum=self.opener_checksum) + key=self.data_manager_key, + opener=Address(uri=self.opener_storage_address, checksum=self.opener_checksum), + archived=False, ) def _setup_data_samples(self): diff --git a/backend/substrapp/tests/compute_tasks/test_context.py b/backend/substrapp/tests/compute_tasks/test_context.py index e69de29bb..147af8975 100644 --- a/backend/substrapp/tests/compute_tasks/test_context.py +++ b/backend/substrapp/tests/compute_tasks/test_context.py @@ -0,0 +1,9 @@ +import pytest + +from substrapp.compute_tasks.errors import InvalidContextError + + +def test_using_archived_datamanager_raises(archived_datamanager_task_input_context): + """Ensure using an archived datamanager in a compute task will raise an invalid context error""" + with pytest.raises(InvalidContextError): + archived_datamanager_task_input_context.data_manager() From 412c9dc3eca4bc096959e8e741f5eb076665fea6 Mon Sep 17 00:00:00 2001 From: Camille Boillet Date: Fri, 7 Oct 2022 12:13:14 +0200 Subject: [PATCH 2/4] fix migration order Signed-off-by: Camille Boillet --- ... => 0040_alter_computeplan_failed_task_category_and_more.py} | 2 +- ...archived_datamanager.py => 0041_add_archived_datamanager.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename backend/api/migrations/{0041_alter_computeplan_failed_task_category_and_more.py => 0040_alter_computeplan_failed_task_category_and_more.py} (96%) rename backend/api/migrations/{0040_add_archived_datamanager.py => 0041_add_archived_datamanager.py} (83%) diff --git a/backend/api/migrations/0041_alter_computeplan_failed_task_category_and_more.py b/backend/api/migrations/0040_alter_computeplan_failed_task_category_and_more.py similarity index 96% rename from backend/api/migrations/0041_alter_computeplan_failed_task_category_and_more.py rename to backend/api/migrations/0040_alter_computeplan_failed_task_category_and_more.py index 41fcfd097..69dc07def 100644 --- a/backend/api/migrations/0041_alter_computeplan_failed_task_category_and_more.py +++ b/backend/api/migrations/0040_alter_computeplan_failed_task_category_and_more.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0040_add_archived_datamanager"), + ("api", "0039_computetaskinputasset"), ] operations = [ diff --git a/backend/api/migrations/0040_add_archived_datamanager.py b/backend/api/migrations/0041_add_archived_datamanager.py similarity index 83% rename from backend/api/migrations/0040_add_archived_datamanager.py rename to backend/api/migrations/0041_add_archived_datamanager.py index cfaf2d181..2ad937753 100644 --- a/backend/api/migrations/0040_add_archived_datamanager.py +++ b/backend/api/migrations/0041_add_archived_datamanager.py @@ -7,7 +7,7 @@ class Migration(migrations.Migration): dependencies = [ - ("api", "0039_computetaskinputasset"), + ("api", "0040_alter_computeplan_failed_task_category_and_more"), ] operations = [ From f879ca257ef6b577c2ecb7f052a18df17aa1a985 Mon Sep 17 00:00:00 2001 From: Camille Boillet Date: Fri, 7 Oct 2022 14:47:30 +0200 Subject: [PATCH 3/4] add archived fields to api output + filter test Signed-off-by: Camille Boillet --- backend/api/serializers/datamanager.py | 1 + backend/api/tests/asset_factory.py | 2 ++ backend/api/tests/views/test_views_datamanager.py | 11 ++++++++++- backend/api/views/datamanager.py | 2 ++ backend/api/views/filters_utils.py | 11 ----------- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/backend/api/serializers/datamanager.py b/backend/api/serializers/datamanager.py index 82f4946ac..497e4682b 100644 --- a/backend/api/serializers/datamanager.py +++ b/backend/api/serializers/datamanager.py @@ -30,6 +30,7 @@ class Meta: "owner", "permissions", "type", + "archived", ] def to_representation(self, instance): diff --git a/backend/api/tests/asset_factory.py b/backend/api/tests/asset_factory.py index c556f9070..756e59c7e 100644 --- a/backend/api/tests/asset_factory.py +++ b/backend/api/tests/asset_factory.py @@ -270,6 +270,7 @@ def create_datamanager( owner: str = DEFAULT_OWNER, channel: str = DEFAULT_CHANNEL, public: bool = False, + archived: bool = False ) -> DataManager: if key is None: key = uuid.uuid4() @@ -285,6 +286,7 @@ def create_datamanager( creation_date=timezone.now(), owner=owner, channel=channel, + archived=archived, **get_permissions(owner, public), **get_log_permissions(owner, public), ) diff --git a/backend/api/tests/views/test_views_datamanager.py b/backend/api/tests/views/test_views_datamanager.py index e8b463305..38da2970e 100644 --- a/backend/api/tests/views/test_views_datamanager.py +++ b/backend/api/tests/views/test_views_datamanager.py @@ -45,7 +45,8 @@ def setUp(self): self.previous_level = self.logger.getEffectiveLevel() self.logger.setLevel(logging.ERROR) - data_manager_1 = factory.create_datamanager(name="datamanager foo") + data_manager_1 = factory.create_datamanager(name="datamanager foo", archived=True) + train_data_sample = factory.create_datasample([data_manager_1]) test_data_sample = factory.create_datasample([data_manager_1], test_only=True) # only for retrieve view @@ -91,6 +92,7 @@ def setUp(self): "public": False, "authorized_ids": ["MyOrg1MSP"], }, + "archived": True }, { "key": str(data_manager_2.key), @@ -121,6 +123,7 @@ def setUp(self): "public": False, "authorized_ids": ["MyOrg1MSP"], }, + "archived": False, }, { "key": str(data_manager_3.key), @@ -151,6 +154,7 @@ def setUp(self): "public": False, "authorized_ids": ["MyOrg1MSP"], }, + "archived": False, }, ] @@ -282,6 +286,11 @@ def test_datamanager_list_cross_assets_filters(self): response = self.client.get(f"{self.url}?{params}", **self.extra) self.assertEqual(response.json().get("results"), self.expected_results[:1]) + def test_datamanager_archived_filters(self): + params = urlencode({"archived": True}) + response = self.client.get(f"{self.url}?{params}", **self.extra) + self.assertEqual(response.json().get("results"), self.expected_results[:1]) + def test_datamanager_match(self): """Match datamanager on part of the name.""" params = urlencode({"match": "manager fo"}) diff --git a/backend/api/views/datamanager.py b/backend/api/views/datamanager.py index 4b3118808..f7317149d 100644 --- a/backend/api/views/datamanager.py +++ b/backend/api/views/datamanager.py @@ -3,6 +3,7 @@ from django.db import models from django.urls import reverse from django_filters.rest_framework import BaseInFilter +from django_filters.rest_framework import BooleanFilter from django_filters.rest_framework import CharFilter from django_filters.rest_framework import DateTimeFromToRangeFilter from django_filters.rest_framework import DjangoFilterBackend @@ -140,6 +141,7 @@ class DataManagerFilter(FilterSet): data_sample_key = CharInFilter( field_name="compute_tasks__data_samples__key", distinct=True, label="data_sample_key" ) + archived = BooleanFilter(field_name="archived", distinct=True, label="archived") class Meta: model = DataManager diff --git a/backend/api/views/filters_utils.py b/backend/api/views/filters_utils.py index bcdd26461..b7e259aa4 100644 --- a/backend/api/views/filters_utils.py +++ b/backend/api/views/filters_utils.py @@ -144,14 +144,3 @@ def filter_queryset(self, request, queryset, view): return queryset.filter(**django_filters) - -class ArchiveFilter(BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - # apply filters to queryset - if view.action == "list": - archived = to_bool(request.query_params.get("archived")) - if archived: - return queryset - return queryset.filter(archived=False) - else: - return queryset From a4b47f84727526ce0c647cee519231cb8c3fe394 Mon Sep 17 00:00:00 2001 From: Camille Boillet Date: Fri, 7 Oct 2022 14:55:18 +0200 Subject: [PATCH 4/4] blackify Signed-off-by: Camille Boillet --- backend/api/tests/asset_factory.py | 2 +- backend/api/tests/views/test_views_datamanager.py | 4 ++-- backend/api/views/filters_utils.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/api/tests/asset_factory.py b/backend/api/tests/asset_factory.py index 756e59c7e..5b7b1b128 100644 --- a/backend/api/tests/asset_factory.py +++ b/backend/api/tests/asset_factory.py @@ -270,7 +270,7 @@ def create_datamanager( owner: str = DEFAULT_OWNER, channel: str = DEFAULT_CHANNEL, public: bool = False, - archived: bool = False + archived: bool = False, ) -> DataManager: if key is None: key = uuid.uuid4() diff --git a/backend/api/tests/views/test_views_datamanager.py b/backend/api/tests/views/test_views_datamanager.py index 38da2970e..f5fd54bd2 100644 --- a/backend/api/tests/views/test_views_datamanager.py +++ b/backend/api/tests/views/test_views_datamanager.py @@ -46,7 +46,7 @@ def setUp(self): self.logger.setLevel(logging.ERROR) data_manager_1 = factory.create_datamanager(name="datamanager foo", archived=True) - + train_data_sample = factory.create_datasample([data_manager_1]) test_data_sample = factory.create_datasample([data_manager_1], test_only=True) # only for retrieve view @@ -92,7 +92,7 @@ def setUp(self): "public": False, "authorized_ids": ["MyOrg1MSP"], }, - "archived": True + "archived": True, }, { "key": str(data_manager_2.key), diff --git a/backend/api/views/filters_utils.py b/backend/api/views/filters_utils.py index b7e259aa4..9462df043 100644 --- a/backend/api/views/filters_utils.py +++ b/backend/api/views/filters_utils.py @@ -143,4 +143,3 @@ def filter_queryset(self, request, queryset, view): django_filters[f'metadata_filters__{f["key"]}__icontains'] = f["value"] return queryset.filter(**django_filters) -