From c37d5af3edcf757670c4bd03f3c9a29b8e2cc73c Mon Sep 17 00:00:00 2001 From: WebSepp Date: Thu, 25 Feb 2021 17:16:41 +0100 Subject: [PATCH] Make virus scanning configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have added five new decision points covering all places in workflow where ClamAV is being used. The processing_config module has been refactor to acommodate for this new use case where a single user-facing decision translates into multiple pre-configured choices in the processing configuration XML document. Co-authored-by: Jesús García Crespo --- .gitignore | 1 + src/MCPServer/lib/assets/workflow.json | 284 +++++++++- src/MCPServer/lib/server/processing_config.py | 501 ++++++++++-------- src/MCPServer/lib/server/rpc_server.py | 4 +- src/MCPServer/lib/server/shared_dirs.py | 260 +++++---- src/MCPServer/tests/test_processing_config.py | 396 +++++++++++++- .../src/components/administration/forms.py | 238 +++------ src/dashboard/src/contrib/mcp/client.py | 3 +- 8 files changed, 1155 insertions(+), 532 deletions(-) diff --git a/.gitignore b/.gitignore index 9bc48e10f9..5bd2242d36 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.mo # IDE specific +.idea *.directory .project .pydevproject diff --git a/src/MCPServer/lib/assets/workflow.json b/src/MCPServer/lib/assets/workflow.json index a0bea6576d..f8f4d2cc9e 100644 --- a/src/MCPServer/lib/assets/workflow.json +++ b/src/MCPServer/lib/assets/workflow.json @@ -117,6 +117,18 @@ }, "link_id": "89071669-3bb6-4e03-90a3-3c8b20c7f6fe" }, + "1ac7d792-b63f-46e0-9945-d48d9e5c02c9": { + "description": { + "en": "Yes", + "es": "Sí", + "fr": "Oui", + "ja": "はい", + "no": "Ja", + "pt_BR": "Sim", + "sv": "Ja" + }, + "link_id": "7d33f228-0fa8-4f4c-a66b-24f8e264c214" + }, "1b04ec43-055c-43b7-9543-bd03c6a778ba": { "description": { "en": "Reject transfer", @@ -143,13 +155,13 @@ }, "1e0df175-d56d-450d-8bee-7df1dc7ae815": { "description": { - "en": "Approve", - "es": "Aprobar", - "fr": "Approuver", - "ja": "承認", - "no": "Godkjenn", - "pt_BR": "Confirmar", - "sv": "Godkänn" + "en": "Yes", + "es": "Sí", + "fr": "Oui", + "ja": "はい", + "no": "Ja", + "pt_BR": "Sim", + "sv": "Ja" }, "link_id": "0f0c1f33-29f2-49ae-b413-3e043da5df61" }, @@ -209,6 +221,18 @@ }, "link_id": "36609513-6502-4aca-886a-6c4ae03a9f05" }, + "34944d4f-762e-4262-8c79-b9fd48521ca0": { + "description": { + "en": "Yes", + "es": "Sí", + "fr": "Oui", + "ja": "はい", + "no": "Ja", + "pt_BR": "Sim", + "sv": "Ja" + }, + "link_id": "21d6d597-b876-4b3f-ab85-f97356f10507" + }, "35151db8-3a11-4b49-8865-f6697ef0ac75": { "description": { "en": "Yes", @@ -271,6 +295,18 @@ }, "link_id": "dae3c416-a8c2-4515-9081-6dbd7b265388" }, + "3e8c0c39-3f30-4c9b-a449-85eef1b2a458": { + "description": { + "en": "Yes", + "es": "Sí", + "fr": "Oui", + "ja": "はい", + "no": "Ja", + "pt_BR": "Sim", + "sv": "Ja" + }, + "link_id": "1ba589db-88d1-48cf-bb1a-a5f9d2b17378" + }, "4171636c-e013-4ecc-ae45-60b5458c208b": { "description": { "en": "Transfers In progress", @@ -394,6 +430,26 @@ }, "link_id": "8f639582-8881-4a8b-8574-d2f86dc4db3d" }, + "63767e4b-9ce8-4fe2-8724-65cc1f763de0": { + "description": { + "en": "No", + "fr": "Non", + "ja": "いいえ", + "no": "Nei", + "sv": "Nej" + }, + "link_id": "b2444a6e-c626-4487-9abc-1556dd89a8ae" + }, + "63be6081-bee8-4cf5-a453-91893e31940f": { + "description": { + "en": "No", + "fr": "Non", + "ja": "いいえ", + "no": "Nei", + "sv": "Nej" + }, + "link_id": "c8f7bf7b-d903-42ec-bfdf-74d357ac4230" + }, "65273f18-5b4e-4944-af4f-09be175a88e8": { "description": { "en": "No", @@ -416,6 +472,28 @@ }, "link_id": "b4567e89-9fea-4256-99f5-a88987026488" }, + "697c0883-798d-4af7-b8b6-101c7f709cd5": { + "description": { + "en": "No", + "fr": "Non", + "ja": "いいえ", + "no": "Nei", + "sv": "Nej" + }, + "link_id": "aaa929e4-5c35-447e-816a-033a66b9b90b" + }, + "6e431096-c403-4cbf-a59a-a26e86be54a8": { + "description": { + "en": "Yes", + "es": "Sí", + "fr": "Oui", + "ja": "はい", + "no": "Ja", + "pt_BR": "Sim", + "sv": "Ja" + }, + "link_id": "8bc92801-4308-4e3b-885b-1a89fdcd3014" + }, "6eb8ebe7-fab3-4e4c-b9d7-14de17625baa": { "description": { "en": "Do not upload DIP", @@ -457,6 +535,16 @@ }, "link_id": "54b73077-a062-41cc-882c-4df1eba447d9" }, + "77355172-b437-4324-9dcc-e2607ad27cb1": { + "description": { + "en": "No", + "fr": "Non", + "ja": "いいえ", + "no": "Nei", + "sv": "Nej" + }, + "link_id": "559d9b14-05bf-4136-918a-de74a821b759" + }, "79f1f5af-7694-48a4-b645-e42790bbf870": { "description": { "en": "No", @@ -467,6 +555,16 @@ }, "link_id": "f8ef02c4-f585-4b0d-9b6f-3cef6fbe527f" }, + "7f5244fe-590b-4e38-beaf-0cf1ccb9e71b": { + "description": { + "en": "No", + "fr": "Non", + "ja": "いいえ", + "no": "Nei", + "sv": "Nej" + }, + "link_id": "087d27be-c719-47d8-9bbb-9a7d8b609c44" + }, "816f28cd-6af1-4d26-97f3-e61645eb881b": { "description": { "en": "baggitDirectory Transfers In progress", @@ -542,6 +640,18 @@ }, "link_id": "accea2bf-ba74-4a3a-bb97-614775c74459" }, + "97be337c-ff27-4869-bf63-ef1abc9df15d": { + "description": { + "en": "Yes", + "es": "Sí", + "fr": "Oui", + "ja": "はい", + "no": "Ja", + "pt_BR": "Sim", + "sv": "Ja" + }, + "link_id": "1c2550f1-3fc0-45d8-8bc4-4c06d720283b" + }, "9918b64c-b898-407b-bce4-a65aa3c11b89": { "description": { "en": "AIP reingest approval chain", @@ -554,12 +664,13 @@ }, "9efab23c-31dc-4cbd-a39d-bb1665460cbe": { "description": { - "en": "Store AIP", - "fr": "Stocker l’AIP", - "ja": "AIPを蓄える", - "no": "Lagre AIP", - "pt_BR": "Armazenar AIP", - "sv": "Lagra AIP" + "en": "Yes", + "es": "Sí", + "fr": "Oui", + "ja": "はい", + "no": "Ja", + "pt_BR": "Sim", + "sv": "Ja" }, "link_id": "49cbcc4d-067b-4cd5-b52e-faf50857b35a" }, @@ -1949,7 +2060,7 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "1ba589db-88d1-48cf-bb1a-a5f9d2b17378" + "link_id": "97a5ddc0-d4e0-43ac-a571-9722405a0a9b" } }, "fallback_job_status": "Failed", @@ -2605,7 +2716,7 @@ "stdout_file": null }, "description": { - "en": "Scan for viruses", + "en": "Scan for viruses in directories", "fr": "Lancer l’antivirus", "ja": "ウイルスをスキャンする", "no": "Søk etter virus", @@ -2705,6 +2816,31 @@ "sv": "Normalisera" } }, + "1dad74a2-95df-4825-bbba-dca8b91d2371": { + "config": { + "@manager": "linkTaskManagerChoice", + "@model": "MicroServiceChainChoice", + "chain_choices": [ + "1ac7d792-b63f-46e0-9945-d48d9e5c02c9", + "697c0883-798d-4af7-b8b6-101c7f709cd5" + ] + }, + "description": { + "en": "Do you want to scan for viruses in extracted files?" + }, + "exit_codes": {}, + "fallback_job_status": "Failed", + "fallback_link_id": null, + "group": { + "en": "Extract packages", + "es": "Extraer paquetes", + "fr": "Extraire les paquets ", + "ja": "パッケージの抽出", + "no": "Pakk ut pakker", + "pt_BR": "Extrair pacotes", + "sv": "Extrahera paket" + } + }, "1dce8e21-7263-4cc4-aa59-968d9793b5f2": { "config": { "@manager": "linkTaskManagerFiles", @@ -2949,7 +3085,7 @@ "stdout_file": null }, "description": { - "en": "Scan for viruses", + "en": "Scan for viruses in attachments", "fr": "Lancer l’antivirus", "ja": "ウイルスをスキャンする", "no": "Søk etter virus", @@ -4169,6 +4305,30 @@ "sv": "Förbered DIP" } }, + "390d6507-5029-4dae-bcd4-ce7178c9b560": { + "config": { + "@manager": "linkTaskManagerChoice", + "@model": "MicroServiceChainChoice", + "chain_choices": [ + "34944d4f-762e-4262-8c79-b9fd48521ca0", + "63be6081-bee8-4cf5-a453-91893e31940f" + ] + }, + "description": { + "en": "Do you want to scan for viruses in attachments?" + }, + "exit_codes": {}, + "fallback_job_status": "Failed", + "fallback_link_id": null, + "group": { + "en": "Scan for viruses", + "fr": "Lancer l’antivirus", + "ja": "ウイルスをスキャンする", + "no": "Søk etter virus", + "pt_BR": "Procurar por vírus", + "sv": "Sök efter virus" + } + }, "39a128e3-c35d-40b7-9363-87f75091e1ff": { "config": { "@manager": "linkTaskManagerDirectories", @@ -5206,11 +5366,11 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "1c2550f1-3fc0-45d8-8bc4-4c06d720283b" + "link_id": "7e81f94e-6441-4430-a12d-76df09181b66" } }, "fallback_job_status": "Failed", - "fallback_link_id": "1c2550f1-3fc0-45d8-8bc4-4c06d720283b", + "fallback_link_id": "7e81f94e-6441-4430-a12d-76df09181b66", "group": { "en": "Approve transfer", "es": "Aprobar transferencia", @@ -5700,7 +5860,7 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "1c2550f1-3fc0-45d8-8bc4-4c06d720283b" + "link_id": "7e81f94e-6441-4430-a12d-76df09181b66" } }, "fallback_job_status": "Failed", @@ -7210,6 +7370,30 @@ "sv": "Skapa SIP från överföringspaket" } }, + "7e81f94e-6441-4430-a12d-76df09181b66": { + "config": { + "@manager": "linkTaskManagerChoice", + "@model": "MicroServiceChainChoice", + "chain_choices": [ + "97be337c-ff27-4869-bf63-ef1abc9df15d", + "77355172-b437-4324-9dcc-e2607ad27cb1" + ] + }, + "description": { + "en": "Do you want to scan for viruses in directories?" + }, + "exit_codes": {}, + "fallback_job_status": "Failed", + "fallback_link_id": null, + "group": { + "en": "Scan for viruses", + "fr": "Lancer l’antivirus", + "ja": "ウイルスをスキャンする", + "no": "Søk etter virus", + "pt_BR": "Procurar por vírus", + "sv": "Sök efter virus" + } + }, "7f975ba6-2185-434c-b507-2911f3c77213": { "config": { "@manager": "linkTaskManagerReplacementDicFromChoice", @@ -7391,6 +7575,29 @@ "sv": "Återingestera AIP" } }, + "856d2d65-cd25-49fa-8da9-cabb78292894": { + "config": { + "@manager": "linkTaskManagerChoice", + "@model": "MicroServiceChainChoice", + "chain_choices": [ + "6e431096-c403-4cbf-a59a-a26e86be54a8", + "63767e4b-9ce8-4fe2-8724-65cc1f763de0" + ] + }, + "description": { + "en": "Do you want to scan for viruses in metadata?" + }, + "exit_codes": {}, + "fallback_job_status": "Failed", + "fallback_link_id": null, + "group": { + "en": "Process metadata directory", + "es": "Procesar directorio de metadatos", + "no": "Prosesser metadatamappe", + "pt_BR": "Processar diretório de metadados", + "sv": "Bearbeta metadatamapp" + } + }, "873b428f-2c86-42b6-b463-aeda925bf559": { "config": { "@manager": "linkTaskManagerDirectories", @@ -8295,6 +8502,29 @@ "sv": "Rensa namn" } }, + "97a5ddc0-d4e0-43ac-a571-9722405a0a9b": { + "config": { + "@manager": "linkTaskManagerChoice", + "@model": "MicroServiceChainChoice", + "chain_choices": [ + "3e8c0c39-3f30-4c9b-a449-85eef1b2a458", + "7f5244fe-590b-4e38-beaf-0cf1ccb9e71b" + ] + }, + "description": { + "en": "Do you want to scan for viruses in submission documentation?" + }, + "exit_codes": {}, + "fallback_job_status": "Failed", + "fallback_link_id": null, + "group": { + "en": "Process submission documentation", + "fr": "Traiter la documentation de soumission", + "no": "Prosesser innsendingsdokumentasjon", + "pt_BR": "Processar documentação de submissão", + "sv": "Bearbeta leveransdokumentation" + } + }, "998044bb-6260-452f-a742-cfb19e80125b": { "config": { "@manager": "linkTaskManagerChoice", @@ -9205,7 +9435,7 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "8bc92801-4308-4e3b-885b-1a89fdcd3014" + "link_id": "856d2d65-cd25-49fa-8da9-cabb78292894" } }, "fallback_job_status": "Failed", @@ -9277,11 +9507,11 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "21d6d597-b876-4b3f-ab85-f97356f10507" + "link_id": "390d6507-5029-4dae-bcd4-ce7178c9b560" } }, "fallback_job_status": "Failed", - "fallback_link_id": "21d6d597-b876-4b3f-ab85-f97356f10507", + "fallback_link_id": "390d6507-5029-4dae-bcd4-ce7178c9b560", "group": { "en": "Extract attachments", "es": "Extraer ficheros adjuntos", @@ -9933,11 +10163,11 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "7d33f228-0fa8-4f4c-a66b-24f8e264c214" + "link_id": "1dad74a2-95df-4825-bbba-dca8b91d2371" } }, "fallback_job_status": "Failed", - "fallback_link_id": "7d33f228-0fa8-4f4c-a66b-24f8e264c214", + "fallback_link_id": "1dad74a2-95df-4825-bbba-dca8b91d2371", "group": { "en": "Extract packages", "es": "Extraer paquetes", @@ -12235,7 +12465,7 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "1c2550f1-3fc0-45d8-8bc4-4c06d720283b" + "link_id": "7e81f94e-6441-4430-a12d-76df09181b66" } }, "fallback_job_status": "Failed", @@ -12737,11 +12967,11 @@ "exit_codes": { "0": { "job_status": "Completed successfully", - "link_id": "1c2550f1-3fc0-45d8-8bc4-4c06d720283b" + "link_id": "7e81f94e-6441-4430-a12d-76df09181b66" } }, "fallback_job_status": "Failed", - "fallback_link_id": "1c2550f1-3fc0-45d8-8bc4-4c06d720283b", + "fallback_link_id": "7e81f94e-6441-4430-a12d-76df09181b66", "group": { "en": "TRIM transfer", "fr": "Transfert TRIM", diff --git a/src/MCPServer/lib/server/processing_config.py b/src/MCPServer/lib/server/processing_config.py index 075f21299c..7835397707 100644 --- a/src/MCPServer/lib/server/processing_config.py +++ b/src/MCPServer/lib/server/processing_config.py @@ -8,250 +8,303 @@ """ from __future__ import absolute_import, division, print_function, unicode_literals +import abc import logging import os import shutil -from collections import OrderedDict from django.conf import settings from lxml import etree +import six from server.workflow_abilities import choice_is_available +import storageService as storage_service logger = logging.getLogger("archivematica.mcp.server.processing_config") -# Types of processing fields: -# - "boolean" (required: "yes_option", "no_option") -# - "storage_service" (required: "purpose") -# - "replace_dict" -# - "chain_choice" (optional: "ignored_choices", "find_duplicates") -processing_fields = OrderedDict() -processing_fields["bd899573-694e-4d33-8c9b-df0af802437d"] = { - "type": "boolean", - "name": "assign_uuids_to_directories", - "yes_option": "2dc3f487-e4b0-4e07-a4b3-6216ed24ca14", - "no_option": "891f60d0-1ba8-48d3-b39e-dd0934635d29", -} -processing_fields["56eebd45-5600-4768-a8c2-ec0114555a3d"] = { - "type": "boolean", - "name": "tree", - "yes_option": "df54fec1-dae1-4ea6-8d17-a839ee7ac4a7", - "no_option": "e9eaef1e-c2e0-4e3b-b942-bfb537162795", -} -processing_fields["f09847c2-ee51-429a-9478-a860477f6b8d"] = { - "type": "replace_dict", - "name": "select_format_id_tool_transfer", -} -processing_fields["dec97e3c-5598-4b99-b26e-f87a435a6b7f"] = { - "type": "chain_choice", - "name": "extract_packages", -} -processing_fields["f19926dd-8fb5-4c79-8ade-c83f61f55b40"] = { - "type": "replace_dict", - "name": "delete_packages", -} -processing_fields["70fc7040-d4fb-4d19-a0e6-792387ca1006"] = { - "type": "boolean", - "name": "policy_checks_originals", - "yes_option": "c611a6ff-dfdb-46d1-b390-f366a6ea6f66", - "no_option": "3e891cc4-39d2-4989-a001-5107a009a223", -} -processing_fields["accea2bf-ba74-4a3a-bb97-614775c74459"] = { - "type": "chain_choice", - "name": "examine", -} -processing_fields["bb194013-597c-4e4a-8493-b36d190f8717"] = { - "type": "chain_choice", - "name": "create_sip", - "ignored_choices": ["Reject transfer"], -} -processing_fields["7a024896-c4f7-4808-a240-44c87c762bc5"] = { - "type": "replace_dict", - "name": "select_format_id_tool_ingest", -} -processing_fields["cb8e5706-e73f-472f-ad9b-d1236af8095f"] = { - "type": "chain_choice", - "name": "normalize", - "ignored_choices": ["Reject SIP"], - "find_duplicates": True, - "label": "Normalize", -} -processing_fields["de909a42-c5b5-46e1-9985-c031b50e9d30"] = { - "type": "boolean", - "name": "normalize_transfer", - "yes_option": "1e0df175-d56d-450d-8bee-7df1dc7ae815", -} -processing_fields["498f7a6d-1b8c-431a-aa5d-83f14f3c5e65"] = { - "type": "replace_dict", - "name": "normalize_thumbnail_mode", -} -processing_fields["153c5f41-3cfb-47ba-9150-2dd44ebc27df"] = { - "type": "boolean", - "name": "policy_checks_preservation_derivatives", - "yes_option": "3a55f688-eca3-4ebc-a012-4ce68290e7b0", - "no_option": "b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546", -} -processing_fields["8ce07e94-6130-4987-96f0-2399ad45c5c2"] = { - "type": "boolean", - "name": "policy_checks_access_derivatives", - "yes_option": "d9760427-b488-4381-832a-de10106de6fe", - "no_option": "76befd52-14c3-44f9-838f-15a4e01624b0", -} -processing_fields["a2ba5278-459a-4638-92d9-38eb1588717d"] = { - "type": "boolean", - "name": "bind_pids", - "yes_option": "8f9dceb5-b978-43e0-a364-8b317a3ac43b", - "no_option": "44a7c397-8187-4fd2-b8f7-c61737c4df49", -} -processing_fields["d0dfa5fc-e3c2-4638-9eda-f96eea1070e0"] = { - "type": "boolean", - "name": "normative_structmap", - "yes_option": "29881c21-3548-454a-9637-ebc5fd46aee0", - "no_option": "65273f18-5b4e-4944-af4f-09be175a88e8", -} -processing_fields["eeb23509-57e2-4529-8857-9d62525db048"] = { - "type": "chain_choice", - "name": "reminder", -} -processing_fields["82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6"] = { - "type": "boolean", - "name": "transcribe_file", - "yes_option": "35151db8-3a11-4b49-8865-f6697ef0ac75", - "no_option": "0a24787c-00e3-4710-b324-90e792bfb484", -} -processing_fields["087d27be-c719-47d8-9bbb-9a7d8b609c44"] = { - "type": "replace_dict", - "name": "select_format_id_tool_submissiondocs", -} -processing_fields["01d64f58-8295-4b7b-9cab-8f1b153a504f"] = { - "type": "replace_dict", - "name": "compression_algo", -} -processing_fields["01c651cb-c174-4ba4-b985-1d87a44d6754"] = { - "type": "replace_dict", - "name": "compression_level", -} -processing_fields["2d32235c-02d4-4686-88a6-96f4d6c7b1c3"] = { - "type": "boolean", - "name": "store_aip", - "yes_option": "9efab23c-31dc-4cbd-a39d-bb1665460cbe", -} -processing_fields["b320ce81-9982-408a-9502-097d0daa48fa"] = { - "type": "storage_service", - "name": "store_aip_location", - "purpose": "AS", -} -processing_fields["92879a29-45bf-4f0b-ac43-e64474f0f2f9"] = { - "type": "chain_choice", - "name": "upload_dip", -} -processing_fields["5e58066d-e113-4383-b20b-f301ed4d751c"] = { - "type": "chain_choice", - "name": "store_dip", -} -processing_fields["cd844b6e-ab3c-4bc6-b34f-7103f88715de"] = { - "type": "storage_service", - "name": "store_dip_location", - "purpose": "DS", -} - - -def get_processing_fields(workflow): - """Return the list of known processing configuration fields. - - It uses `processing_fields`` defined in this module as a base and extended - after some workflow lookups. - """ - for link_id, config in processing_fields.items(): - link = workflow.get_link(link_id) - if config["type"] == "replace_dict": - config["options"] = _get_options_for_replace_dict(link) - elif config["type"] == "chain_choice": - config["options"] = _get_options_for_chain_choice( - link, workflow, config.get("ignored_choices", []) +@six.add_metaclass(abc.ABCMeta) +class ProcessingConfigField(object): + def __init__(self, link_id, name, **kwargs): + self.link_id = link_id + self.name = name + self.options = self.read_config(kwargs) + + @abc.abstractmethod + def read_config(self, options): + """Implementors must use this method to process additional config.""" + + @abc.abstractmethod + def add_choices(self, workflow, lang): + """Implementors must use this method to add field choices.""" + + def to_dict(self, workflow, lang): + """It generates a dictionary with all the information needed to feed + a drop-down, including its choices and where they apply in workflow + which can be more than a single entry. + + E.g., "Normalize for preservation" is a chain link shared across more + than a single decision point. + """ + self.link = workflow.get_link(self.link_id) + self.choices = [] + self.add_choices(workflow, lang) + return { + "id": self.link.id, + "name": self.name, + "label": self.link.get_label("description", lang), + "choices": self.choices, + } + + +class StorageLocationField(ProcessingConfigField): + ALLOWED_PURPOSES = ("AS", "DS") + + def read_config(self, options): + self.purpose = options["purpose"] + if self.purpose not in self.ALLOWED_PURPOSES: + raise ValueError( + "Purpose %s is invalid; valid purposes: %s." + % (self.purpose, ", ".join(self.ALLOWED_PURPOSES)) ) - _populate_duplicates_chain_choice(workflow, link, config) - return processing_fields - - -def _get_options_for_replace_dict(link): - return [ - (item["id"], item["description"]["en"]) for item in link.config["replacements"] - ] - - -def _get_options_for_chain_choice(link, workflow, ignored_choices): - ret = [] - for chain_id in link.config["chain_choices"]: - chain = workflow.get_chain(chain_id) - label = chain.get_label("description") - if label in ignored_choices: - continue - if not choice_is_available(link, chain): - continue - ret.append((chain_id, label)) - return ret - - -def _populate_duplicates_chain_choice(workflow, link, config): - """Find and populate chain choice duplicates. - - When the user chooses a value like "Normalize for preservation" in the - "Normalize" processing config, this function makes sure that all the - matching chain links are listed so the user choise applies to all of them. - - Given the following config item (see `processing_fields` in this module): - - config[find_duplicates] = True - config[label] = "Normalize" - config[options] = [ - (2b93cecd4-71f2-4e28-bc39-d32fd62c5a94", "Normalize ...") - (2612e3609-ce9a-4df6-a9a3-63d634d2d934", ...) - (2c34bd22a-d077-4180-bf58-01db35bdb644", ...) - (289cb80dd-0636-464f-930d-57b61e3928b2", ...) - (2a6ed697e-6189-4b4e-9f80-29209abc7937", ...) - (2e600b56d-1a43-4031-9d7c-f64f123e5662", ...) - (2fb7a326e-1e50-4b48-91b9-4917ff8d0ae8", ...) - ] - This function populates a new property with a list of matching links, e.g.: + def add_choices(self, workflow, lang): + value = "/api/v2/location/default/%s/" % self.purpose + self.choices.append( + { + "value": value, + "label": "Default location", + "applies_to": [(self.link.id, value, "Default location")], + } + ) + + locations = self.get_storage_locations() + if locations: + for loc in locations: + label = loc["description"] or loc["relative_path"] + self.choices.append( + { + "value": loc["resource_uri"], + "label": label, + "applies_to": [(self.link.id, loc["resource_uri"], label)], + } + ) + + def get_storage_locations(self): + return storage_service.get_location(purpose=self.purpose) + + +class ReplaceDictField(ProcessingConfigField): + def read_config(self, options): + pass + + def add_choices(self, workflow, lang): + for item in self.link.config["replacements"]: + label = item["description"].get_label(lang) + self.choices.append( + { + "value": item["id"], + "label": label, + "applies_to": [(self.link.id, item["id"], label)], + } + ) - config[duplicates] = { - = (chain_desc, (, ), ...) - ... - } - It's used in `administration/forms.py` so we can build configs like the - following when the user picks "Normalize for preservation" which has more - than one match: - - - - cb8e5706-e73f-472f-ad9b-d1236af8095f - 612e3609-ce9a-4df6-a9a3-63d634d2d934 - - - - 7509e7dc-1e1b-4dce-8d21-e130515fce73 - 612e3609-ce9a-4df6-a9a3-63d634d2d934 - +class ChainChoicesField(ProcessingConfigField): + """Populate choices based on the list of chains indicated by + ``chain_choices`` in the workflow link definition. + + ``ignored_choices`` (List[str]) is an optional list of chain names that will + not be incorporated. ``find_duplicates`` is an optional string used to match + all links making use of that choice, e.g. "Normalize for preservation". """ - if not config.get("find_duplicates", False): - return - config["duplicates"] = {} - for chain_id, chain_desc in config["options"]: - results = [] - for link in workflow.get_links().values(): - if config["label"] != link.get_label("description"): + + def read_config(self, options): + self.ignored_choices = options.get("ignored_choices", []) + self.find_duplicates = options.get("find_duplicates") + + def add_choices(self, workflow, lang): + for chain_id in self.link.config["chain_choices"]: + chain = workflow.get_chain(chain_id) + chain_desc = chain.get_label("description") + if chain_desc in self.ignored_choices: + continue + if not choice_is_available(self.link, chain): continue - for cid in link.config["chain_choices"]: - chain = workflow.get_chain(cid) - if chain_desc != chain.get_label("description"): + self.choices.append( + { + "value": chain_id, + "label": chain.get_label("description", lang), + "applies_to": [(self.link_id, chain_id, chain_desc)], + } + ) + if not self.find_duplicates: + continue + for link in workflow.get_links().values(): + if link.id == self.link_id: continue - results.append((link.id, chain.id)) - config["duplicates"][chain_id] = (chain_desc, results) + if link.get_label("description") != self.find_duplicates: + continue + for cid in link.config["chain_choices"]: + chain = workflow.get_chain(cid) + if chain_desc != chain.get_label("description"): + continue + self.choices[-1]["applies_to"].append( + (link.id, chain.id, chain_desc) + ) + + +class SharedChainChoicesField(ProcessingConfigField): + """Populate choices that are equivalent across multiple chain links. + + Use `related_links` (List[str]) to indicate additional link identifiers. + """ + + def read_config(self, options): + self.related_links = options.get("related_links", []) + + def add_choices(self, workflow, lang): + # Full list of choices based on the master link. + choices = [ + workflow.get_chain(chain_id)["description"]["en"] + for chain_id in self.link.config["chain_choices"] + ] + + # All link identifiers. + link_ids = [self.link.id] + self.related_links + + # Each choice capturing the underlying chain choices for each link, + for choice in choices: + applies_to = [] + value = None + for link_id in link_ids: + link = workflow.get_link(link_id) + for chain_id in link.config["chain_choices"]: + chain = workflow.get_chain(chain_id) + if chain.get_label("description") == choice: + applies_to.append((link_id, chain_id, choice)) + if link_id == self.link.id: + value = chain_id + self.choices.append( + {"value": value, "label": choice, "applies_to": applies_to} + ) + + +# A list of processing configuration fields that we want to display via the +# web user interface. Use one of the supported configuration classes, i.e. all +# classes extending ``ProcessingConfigField``. +processing_fields = [ + ReplaceDictField( + link_id="bd899573-694e-4d33-8c9b-df0af802437d", + name="assign_uuids_to_directories", + ), + ChainChoicesField( + link_id="56eebd45-5600-4768-a8c2-ec0114555a3d", + name="generate_transfer_structure", + ), + ReplaceDictField( + link_id="f09847c2-ee51-429a-9478-a860477f6b8d", + name="select_format_id_tool_transfer", + ), + ChainChoicesField( + link_id="dec97e3c-5598-4b99-b26e-f87a435a6b7f", name="extract_packages" + ), + ReplaceDictField( + link_id="f19926dd-8fb5-4c79-8ade-c83f61f55b40", name="delete_packages" + ), + ChainChoicesField( + link_id="70fc7040-d4fb-4d19-a0e6-792387ca1006", name="policy_checks_originals" + ), + ChainChoicesField( + link_id="accea2bf-ba74-4a3a-bb97-614775c74459", name="examine_contents" + ), + ChainChoicesField( + link_id="bb194013-597c-4e4a-8493-b36d190f8717", + name="create_sip", + ignored_choices=["Reject transfer"], + ), + ReplaceDictField( + link_id="7a024896-c4f7-4808-a240-44c87c762bc5", + name="select_format_id_tool_ingest", + ), + ChainChoicesField( + link_id="cb8e5706-e73f-472f-ad9b-d1236af8095f", + name="normalize", + ignored_choices=["Reject SIP"], + find_duplicates="Normalize", + ), + ChainChoicesField( + link_id="de909a42-c5b5-46e1-9985-c031b50e9d30", + name="normalize_transfer", + ignored_choices=["Redo", "Reject"], + ), + ReplaceDictField( + link_id="498f7a6d-1b8c-431a-aa5d-83f14f3c5e65", name="normalize_thumbnail_mode" + ), + ChainChoicesField( + link_id="153c5f41-3cfb-47ba-9150-2dd44ebc27df", + name="policy_checks_preservation_derivatives", + ), + ChainChoicesField( + link_id="8ce07e94-6130-4987-96f0-2399ad45c5c2", + name="policy_checks_access_derivatives", + ), + ChainChoicesField(link_id="a2ba5278-459a-4638-92d9-38eb1588717d", name="bind_pids"), + ChainChoicesField( + link_id="d0dfa5fc-e3c2-4638-9eda-f96eea1070e0", name="normative_structmap" + ), + ChainChoicesField(link_id="eeb23509-57e2-4529-8857-9d62525db048", name="reminder"), + ChainChoicesField( + link_id="82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6", name="transcribe_file" + ), + ReplaceDictField( + link_id="087d27be-c719-47d8-9bbb-9a7d8b609c44", + name="select_format_id_tool_submissiondocs", + ), + ReplaceDictField( + link_id="01d64f58-8295-4b7b-9cab-8f1b153a504f", name="compression_algo" + ), + ReplaceDictField( + link_id="01c651cb-c174-4ba4-b985-1d87a44d6754", name="compression_level" + ), + ChainChoicesField( + link_id="2d32235c-02d4-4686-88a6-96f4d6c7b1c3", + name="store_aip", + ignored_choices=["Reject AIP"], + ), + StorageLocationField( + link_id="b320ce81-9982-408a-9502-097d0daa48fa", + name="store_aip_location", + purpose="AS", + ), + ChainChoicesField( + link_id="92879a29-45bf-4f0b-ac43-e64474f0f2f9", name="upload_dip" + ), + ChainChoicesField(link_id="5e58066d-e113-4383-b20b-f301ed4d751c", name="store_dip"), + StorageLocationField( + link_id="cd844b6e-ab3c-4bc6-b34f-7103f88715de", + name="store_dip_location", + purpose="DS", + ), + SharedChainChoicesField( + link_id="856d2d65-cd25-49fa-8da9-cabb78292894", + name="virus_scanning", + related_links=[ + "1dad74a2-95df-4825-bbba-dca8b91d2371", + "7e81f94e-6441-4430-a12d-76df09181b66", + "390d6507-5029-4dae-bcd4-ce7178c9b560", + "97a5ddc0-d4e0-43ac-a571-9722405a0a9b", + ], + ), +] + + +def get_processing_fields(workflow, lang="en"): + """Return the list dict form of all processing configuration fields defined + in the module-level attribute ``processing_fields``. + """ + return [field.to_dict(workflow, lang) for field in processing_fields] def copy_processing_config(processing_config, destination_path): diff --git a/src/MCPServer/lib/server/rpc_server.py b/src/MCPServer/lib/server/rpc_server.py index c20d08c51e..86d1054805 100644 --- a/src/MCPServer/lib/server/rpc_server.py +++ b/src/MCPServer/lib/server/rpc_server.py @@ -328,14 +328,14 @@ def _approve_partial_reingest_handler(self, worker, job, payload): if job_chain: self.package_queue.schedule_job(next(job_chain)) - def _get_processing_config_fields_handler(self, worker, job): + def _get_processing_config_fields_handler(self, worker, job, payload): """List processing configuration fields. [config] name = getProcessingConfigFields raise_exc = False """ - return get_processing_fields(self.workflow) + return get_processing_fields(self.workflow, payload.get("lang")) def _units_statuses_handler(self, worker, job, payload): """Returns the status of units that are of type SIP or Transfer. diff --git a/src/MCPServer/lib/server/shared_dirs.py b/src/MCPServer/lib/server/shared_dirs.py index 5caff61ede..4a31517808 100644 --- a/src/MCPServer/lib/server/shared_dirs.py +++ b/src/MCPServer/lib/server/shared_dirs.py @@ -16,65 +16,90 @@ # TODO: store this in assets at least DEFAULT_PROCESSING_CONFIG = """ - + - 01c651cb-c174-4ba4-b985-1d87a44d6754 - 414da421-b83f-4648-895f-a34840e3c3f5 - - - - 087d27be-c719-47d8-9bbb-9a7d8b609c44 - 4dec164b-79b0-4459-8505-8095af9655b5 - - - - a2ba5278-459a-4638-92d9-38eb1588717d - 44a7c397-8187-4fd2-b8f7-c61737c4df49 + bd899573-694e-4d33-8c9b-df0af802437d + 891f60d0-1ba8-48d3-b39e-dd0934635d29 - + 56eebd45-5600-4768-a8c2-ec0114555a3d df54fec1-dae1-4ea6-8d17-a839ee7ac4a7 - + 70fc7040-d4fb-4d19-a0e6-792387ca1006 3e891cc4-39d2-4989-a001-5107a009a223 - + + + 7a024896-c4f7-4808-a240-44c87c762bc5 + 3c1faec7-7e1e-4cdd-b3bd-e2f05f4baa9b + + 498f7a6d-1b8c-431a-aa5d-83f14f3c5e65 c318b224-b718-4535-a911-494b1af6ff26 - + - 01d64f58-8295-4b7b-9cab-8f1b153a504f - 9475447c-9889-430c-9477-6287a9574c5b + 153c5f41-3cfb-47ba-9150-2dd44ebc27df + b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546 - + 8ce07e94-6130-4987-96f0-2399ad45c5c2 76befd52-14c3-44f9-838f-15a4e01624b0 - + - 7a024896-c4f7-4808-a240-44c87c762bc5 - 3c1faec7-7e1e-4cdd-b3bd-e2f05f4baa9b + a2ba5278-459a-4638-92d9-38eb1588717d + 44a7c397-8187-4fd2-b8f7-c61737c4df49 - + - 153c5f41-3cfb-47ba-9150-2dd44ebc27df - b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546 + d0dfa5fc-e3c2-4638-9eda-f96eea1070e0 + 65273f18-5b4e-4944-af4f-09be175a88e8 - + - bd899573-694e-4d33-8c9b-df0af802437d - 891f60d0-1ba8-48d3-b39e-dd0934635d29 + 087d27be-c719-47d8-9bbb-9a7d8b609c44 + 4dec164b-79b0-4459-8505-8095af9655b5 - + - d0dfa5fc-e3c2-4638-9eda-f96eea1070e0 - 65273f18-5b4e-4944-af4f-09be175a88e8 + 01d64f58-8295-4b7b-9cab-8f1b153a504f + 9475447c-9889-430c-9477-6287a9574c5b + + + + 01c651cb-c174-4ba4-b985-1d87a44d6754 + 414da421-b83f-4648-895f-a34840e3c3f5 + + + + 856d2d65-cd25-49fa-8da9-cabb78292894 + 6e431096-c403-4cbf-a59a-a26e86be54a8 + + + + 1dad74a2-95df-4825-bbba-dca8b91d2371 + 1ac7d792-b63f-46e0-9945-d48d9e5c02c9 + + + + 7e81f94e-6441-4430-a12d-76df09181b66 + 97be337c-ff27-4869-bf63-ef1abc9df15d + + + + 390d6507-5029-4dae-bcd4-ce7178c9b560 + 34944d4f-762e-4262-8c79-b9fd48521ca0 + + + + 97a5ddc0-d4e0-43ac-a571-9722405a0a9b + 3e8c0c39-3f30-4c9b-a449-85eef1b2a458 @@ -82,140 +107,165 @@ AUTOMATED_PROCESSING_CONFIG = """ - + - 5e58066d-e113-4383-b20b-f301ed4d751c - 8d29eb3d-a8a8-4347-806e-3d8227ed44a1 + bd899573-694e-4d33-8c9b-df0af802437d + 2dc3f487-e4b0-4e07-a4b3-6216ed24ca14 - + - 01c651cb-c174-4ba4-b985-1d87a44d6754 - 414da421-b83f-4648-895f-a34840e3c3f5 + 56eebd45-5600-4768-a8c2-ec0114555a3d + e9eaef1e-c2e0-4e3b-b942-bfb537162795 - + - accea2bf-ba74-4a3a-bb97-614775c74459 - e0a39199-c62a-4a2f-98de-e9d1116460a8 + f09847c2-ee51-429a-9478-a860477f6b8d + d97297c7-2b49-4cfe-8c9f-0613d63ed763 - + - 087d27be-c719-47d8-9bbb-9a7d8b609c44 - 4dec164b-79b0-4459-8505-8095af9655b5 + dec97e3c-5598-4b99-b26e-f87a435a6b7f + 01d80b27-4ad1-4bd1-8f8d-f819f18bf685 - + - cb8e5706-e73f-472f-ad9b-d1236af8095f - 612e3609-ce9a-4df6-a9a3-63d634d2d934 + f19926dd-8fb5-4c79-8ade-c83f61f55b40 + 85b1e45d-8f98-4cae-8336-72f40e12cbef - + - 7509e7dc-1e1b-4dce-8d21-e130515fce73 - 612e3609-ce9a-4df6-a9a3-63d634d2d934 + 70fc7040-d4fb-4d19-a0e6-792387ca1006 + 3e891cc4-39d2-4989-a001-5107a009a223 - + - a2ba5278-459a-4638-92d9-38eb1588717d - 44a7c397-8187-4fd2-b8f7-c61737c4df49 + accea2bf-ba74-4a3a-bb97-614775c74459 + e0a39199-c62a-4a2f-98de-e9d1116460a8 - + bb194013-597c-4e4a-8493-b36d190f8717 61cfa825-120e-4b17-83e6-51a42b67d969 - + - f19926dd-8fb5-4c79-8ade-c83f61f55b40 - 85b1e45d-8f98-4cae-8336-72f40e12cbef + 7a024896-c4f7-4808-a240-44c87c762bc5 + 5b3c8268-5b33-4b70-b1aa-0e4540fe03d1 - + - 82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6 - 0a24787c-00e3-4710-b324-90e792bfb484 + cb8e5706-e73f-472f-ad9b-d1236af8095f + 612e3609-ce9a-4df6-a9a3-63d634d2d934 - + - f09847c2-ee51-429a-9478-a860477f6b8d - d97297c7-2b49-4cfe-8c9f-0613d63ed763 + 7509e7dc-1e1b-4dce-8d21-e130515fce73 + 612e3609-ce9a-4df6-a9a3-63d634d2d934 - + - cd844b6e-ab3c-4bc6-b34f-7103f88715de - /api/v2/location/default/DS/ + de909a42-c5b5-46e1-9985-c031b50e9d30 + 1e0df175-d56d-450d-8bee-7df1dc7ae815 - + - 56eebd45-5600-4768-a8c2-ec0114555a3d - e9eaef1e-c2e0-4e3b-b942-bfb537162795 + 498f7a6d-1b8c-431a-aa5d-83f14f3c5e65 + 972fce6c-52c8-4c00-99b9-d6814e377974 - + - 70fc7040-d4fb-4d19-a0e6-792387ca1006 - 3e891cc4-39d2-4989-a001-5107a009a223 + 153c5f41-3cfb-47ba-9150-2dd44ebc27df + b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546 + + + + 8ce07e94-6130-4987-96f0-2399ad45c5c2 + 76befd52-14c3-44f9-838f-15a4e01624b0 + + + + a2ba5278-459a-4638-92d9-38eb1588717d + 44a7c397-8187-4fd2-b8f7-c61737c4df49 + + + + d0dfa5fc-e3c2-4638-9eda-f96eea1070e0 + 65273f18-5b4e-4944-af4f-09be175a88e8 - + eeb23509-57e2-4529-8857-9d62525db048 5727faac-88af-40e8-8c10-268644b0142d - + - 498f7a6d-1b8c-431a-aa5d-83f14f3c5e65 - 972fce6c-52c8-4c00-99b9-d6814e377974 + 82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6 + 0a24787c-00e3-4710-b324-90e792bfb484 + + + + 087d27be-c719-47d8-9bbb-9a7d8b609c44 + 4dec164b-79b0-4459-8505-8095af9655b5 - + 01d64f58-8295-4b7b-9cab-8f1b153a504f 9475447c-9889-430c-9477-6287a9574c5b - + + + 01c651cb-c174-4ba4-b985-1d87a44d6754 + 414da421-b83f-4648-895f-a34840e3c3f5 + + 2d32235c-02d4-4686-88a6-96f4d6c7b1c3 9efab23c-31dc-4cbd-a39d-bb1665460cbe - + - 8ce07e94-6130-4987-96f0-2399ad45c5c2 - 76befd52-14c3-44f9-838f-15a4e01624b0 + b320ce81-9982-408a-9502-097d0daa48fa + /api/v2/location/default/AS/ - + - 7a024896-c4f7-4808-a240-44c87c762bc5 - 5b3c8268-5b33-4b70-b1aa-0e4540fe03d1 + 92879a29-45bf-4f0b-ac43-e64474f0f2f9 + 6eb8ebe7-fab3-4e4c-b9d7-14de17625baa - + - 153c5f41-3cfb-47ba-9150-2dd44ebc27df - b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546 + 5e58066d-e113-4383-b20b-f301ed4d751c + 8d29eb3d-a8a8-4347-806e-3d8227ed44a1 - + - bd899573-694e-4d33-8c9b-df0af802437d - 2dc3f487-e4b0-4e07-a4b3-6216ed24ca14 + cd844b6e-ab3c-4bc6-b34f-7103f88715de + /api/v2/location/default/DS/ - + - b320ce81-9982-408a-9502-097d0daa48fa - /api/v2/location/default/AS/ + 856d2d65-cd25-49fa-8da9-cabb78292894 + 6e431096-c403-4cbf-a59a-a26e86be54a8 - + - d0dfa5fc-e3c2-4638-9eda-f96eea1070e0 - 65273f18-5b4e-4944-af4f-09be175a88e8 + 1dad74a2-95df-4825-bbba-dca8b91d2371 + 1ac7d792-b63f-46e0-9945-d48d9e5c02c9 - + - dec97e3c-5598-4b99-b26e-f87a435a6b7f - 01d80b27-4ad1-4bd1-8f8d-f819f18bf685 + 7e81f94e-6441-4430-a12d-76df09181b66 + 97be337c-ff27-4869-bf63-ef1abc9df15d - + - de909a42-c5b5-46e1-9985-c031b50e9d30 - 1e0df175-d56d-450d-8bee-7df1dc7ae815 + 390d6507-5029-4dae-bcd4-ce7178c9b560 + 34944d4f-762e-4262-8c79-b9fd48521ca0 - + - 92879a29-45bf-4f0b-ac43-e64474f0f2f9 - 6eb8ebe7-fab3-4e4c-b9d7-14de17625baa + 97a5ddc0-d4e0-43ac-a571-9722405a0a9b + 3e8c0c39-3f30-4c9b-a449-85eef1b2a458 diff --git a/src/MCPServer/tests/test_processing_config.py b/src/MCPServer/tests/test_processing_config.py index 1fd84df29d..a922cdc0fb 100644 --- a/src/MCPServer/tests/test_processing_config.py +++ b/src/MCPServer/tests/test_processing_config.py @@ -1,13 +1,17 @@ +from __future__ import unicode_literals + import os import pytest from server.processing_config import ( - _get_options_for_chain_choice, - _populate_duplicates_chain_choice, get_processing_fields, processing_configuration_file_exists, processing_fields, + StorageLocationField, + ChainChoicesField, + SharedChainChoicesField, + ReplaceDictField, ) from server.workflow import load @@ -24,21 +28,389 @@ def _workflow(): return load(fp) -def test_get_processing_fields(_workflow): +def test_get_processing_fields(mocker, _workflow): + mocker.patch("storageService.get_location", return_value=[]) + fields = get_processing_fields(_workflow) + assert len(fields) == len(processing_fields) -def test__populate_duplicates_chain_choice(_workflow): - link_id = "cb8e5706-e73f-472f-ad9b-d1236af8095f" - link = _workflow.get_link(link_id) - config = processing_fields[link_id] - config["options"] = _get_options_for_chain_choice( - link, _workflow, config.get("ignored_choices") +def test_storage_location_field(mocker, _workflow): + def mocked_get_location(purpose): + return [ + { + "resource_uri": "/api/v2/location/e1452470-a51c-4fd7-b2c1-b217b7dbfa11/", + "relative_path": "mnt/disk1", + "description": "Description %s" % purpose, + } + ] + + mocker.patch("storageService.get_location", side_effect=mocked_get_location) + + mocker.patch( + "server.processing_config.processing_fields", + new=[ + StorageLocationField( + link_id="b320ce81-9982-408a-9502-097d0daa48fa", + name="store_aip_location", + purpose="AS", + ), + StorageLocationField( + link_id="cd844b6e-ab3c-4bc6-b34f-7103f88715de", + name="store_dip_location", + purpose="DS", + ), + ], + ) + + assert get_processing_fields(_workflow) == [ + { + "choices": [ + { + "applies_to": [ + ( + "b320ce81-9982-408a-9502-097d0daa48fa", + "/api/v2/location/default/AS/", + "Default location", + ) + ], + "value": "/api/v2/location/default/AS/", + "label": "Default location", + }, + { + "applies_to": [ + ( + "b320ce81-9982-408a-9502-097d0daa48fa", + "/api/v2/location/e1452470-a51c-4fd7-b2c1-b217b7dbfa11/", + "Description AS", + ) + ], + "value": "/api/v2/location/e1452470-a51c-4fd7-b2c1-b217b7dbfa11/", + "label": "Description AS", + }, + ], + "id": "b320ce81-9982-408a-9502-097d0daa48fa", + "name": "store_aip_location", + "label": "Store AIP location", + }, + { + "choices": [ + { + "applies_to": [ + ( + "cd844b6e-ab3c-4bc6-b34f-7103f88715de", + "/api/v2/location/default/DS/", + "Default location", + ) + ], + "value": "/api/v2/location/default/DS/", + "label": "Default location", + }, + { + "applies_to": [ + ( + "cd844b6e-ab3c-4bc6-b34f-7103f88715de", + "/api/v2/location/e1452470-a51c-4fd7-b2c1-b217b7dbfa11/", + "Description DS", + ) + ], + "value": "/api/v2/location/e1452470-a51c-4fd7-b2c1-b217b7dbfa11/", + "label": "Description DS", + }, + ], + "id": "cd844b6e-ab3c-4bc6-b34f-7103f88715de", + "name": "store_dip_location", + "label": "Store DIP location", + }, + ] + + +def test_replace_dict_field(mocker, _workflow): + mocker.patch( + "server.processing_config.processing_fields", + new=[ + ReplaceDictField( + link_id="f09847c2-ee51-429a-9478-a860477f6b8d", + name="select_format_id_tool_transfer", + ), + ReplaceDictField( + link_id="f19926dd-8fb5-4c79-8ade-c83f61f55b40", name="delete_packages" + ), + ], ) - _populate_duplicates_chain_choice(_workflow, link, config) - duplicates = config["duplicates"] - assert len(duplicates) > 0 + + assert get_processing_fields(_workflow) == [ + { + "choices": [ + { + "applies_to": [ + ( + "f09847c2-ee51-429a-9478-a860477f6b8d", + "d97297c7-2b49-4cfe-8c9f-0613d63ed763", + "Yes", + ) + ], + "value": "d97297c7-2b49-4cfe-8c9f-0613d63ed763", + "label": "Yes", + }, + { + "applies_to": [ + ( + "f09847c2-ee51-429a-9478-a860477f6b8d", + "1f77af0a-2f7a-468f-af8c-653a9e61ca4f", + "No", + ) + ], + "value": "1f77af0a-2f7a-468f-af8c-653a9e61ca4f", + "label": "No", + }, + ], + "id": "f09847c2-ee51-429a-9478-a860477f6b8d", + "name": "select_format_id_tool_transfer", + "label": "Do you want to perform file format identification?", + }, + { + "choices": [ + { + "applies_to": [ + ( + "f19926dd-8fb5-4c79-8ade-c83f61f55b40", + "85b1e45d-8f98-4cae-8336-72f40e12cbef", + "Yes", + ) + ], + "value": "85b1e45d-8f98-4cae-8336-72f40e12cbef", + "label": "Yes", + }, + { + "applies_to": [ + ( + "f19926dd-8fb5-4c79-8ade-c83f61f55b40", + "72e8443e-a8eb-49a8-ba5f-76d52f960bde", + "No", + ) + ], + "value": "72e8443e-a8eb-49a8-ba5f-76d52f960bde", + "label": "No", + }, + ], + "id": "f19926dd-8fb5-4c79-8ade-c83f61f55b40", + "name": "delete_packages", + "label": "Delete package after extraction?", + }, + ] + + +def test_chain_choices_field(mocker, _workflow): + mocker.patch( + "server.processing_config.processing_fields", + new=[ + ChainChoicesField( + link_id="eeb23509-57e2-4529-8857-9d62525db048", name="reminder" + ), + ChainChoicesField( + link_id="cb8e5706-e73f-472f-ad9b-d1236af8095f", + name="normalize", + ignored_choices=["Reject SIP"], + find_duplicates="Normalize", + ), + ], + ) + + assert get_processing_fields(_workflow) == [ + { + "choices": [ + { + "applies_to": [ + ( + "eeb23509-57e2-4529-8857-9d62525db048", + "5727faac-88af-40e8-8c10-268644b0142d", + "Continue", + ) + ], + "value": "5727faac-88af-40e8-8c10-268644b0142d", + "label": "Continue", + } + ], + "id": "eeb23509-57e2-4529-8857-9d62525db048", + "name": "reminder", + "label": "Reminder: add metadata if desired", + }, + { + "choices": [ + { + "applies_to": [ + ( + "cb8e5706-e73f-472f-ad9b-d1236af8095f", + "b93cecd4-71f2-4e28-bc39-d32fd62c5a94", + "Normalize for preservation and access", + ) + ], + "value": "b93cecd4-71f2-4e28-bc39-d32fd62c5a94", + "label": "Normalize for preservation and access", + }, + { + "applies_to": [ + ( + "cb8e5706-e73f-472f-ad9b-d1236af8095f", + "612e3609-ce9a-4df6-a9a3-63d634d2d934", + "Normalize for preservation", + ), + ( + "7509e7dc-1e1b-4dce-8d21-e130515fce73", + "612e3609-ce9a-4df6-a9a3-63d634d2d934", + "Normalize for preservation", + ), + ], + "value": "612e3609-ce9a-4df6-a9a3-63d634d2d934", + "label": "Normalize for preservation", + }, + { + "applies_to": [ + ( + "cb8e5706-e73f-472f-ad9b-d1236af8095f", + "fb7a326e-1e50-4b48-91b9-4917ff8d0ae8", + "Normalize for access", + ) + ], + "value": "fb7a326e-1e50-4b48-91b9-4917ff8d0ae8", + "label": "Normalize for access", + }, + { + "applies_to": [ + ( + "cb8e5706-e73f-472f-ad9b-d1236af8095f", + "e600b56d-1a43-4031-9d7c-f64f123e5662", + "Normalize service files for access", + ) + ], + "value": "e600b56d-1a43-4031-9d7c-f64f123e5662", + "label": "Normalize service files for access", + }, + { + "applies_to": [ + ( + "cb8e5706-e73f-472f-ad9b-d1236af8095f", + "c34bd22a-d077-4180-bf58-01db35bdb644", + "Normalize manually", + ) + ], + "value": "c34bd22a-d077-4180-bf58-01db35bdb644", + "label": "Normalize manually", + }, + { + "applies_to": [ + ( + "cb8e5706-e73f-472f-ad9b-d1236af8095f", + "89cb80dd-0636-464f-930d-57b61e3928b2", + "Do not normalize", + ), + ( + "7509e7dc-1e1b-4dce-8d21-e130515fce73", + "e8544c5e-9cbb-4b8f-a68b-6d9b4d7f7362", + "Do not normalize", + ), + ], + "value": "89cb80dd-0636-464f-930d-57b61e3928b2", + "label": "Do not normalize", + }, + ], + "id": "cb8e5706-e73f-472f-ad9b-d1236af8095f", + "name": "normalize", + "label": "Normalize", + }, + ] + + +def test_shared_choices_field(mocker, _workflow): + mocker.patch( + "server.processing_config.processing_fields", + new=[ + SharedChainChoicesField( + link_id="856d2d65-cd25-49fa-8da9-cabb78292894", + name="virus_scanning", + related_links=[ + "1dad74a2-95df-4825-bbba-dca8b91d2371", + "7e81f94e-6441-4430-a12d-76df09181b66", + "390d6507-5029-4dae-bcd4-ce7178c9b560", + "97a5ddc0-d4e0-43ac-a571-9722405a0a9b", + ], + ) + ], + ) + + assert get_processing_fields(_workflow) == [ + { + "id": "856d2d65-cd25-49fa-8da9-cabb78292894", + "label": "Do you want to scan for viruses in metadata?", + "name": "virus_scanning", + "choices": [ + { + "value": "6e431096-c403-4cbf-a59a-a26e86be54a8", + "label": "Yes", + "applies_to": [ + ( + "856d2d65-cd25-49fa-8da9-cabb78292894", + "6e431096-c403-4cbf-a59a-a26e86be54a8", + "Yes", + ), + ( + "1dad74a2-95df-4825-bbba-dca8b91d2371", + "1ac7d792-b63f-46e0-9945-d48d9e5c02c9", + "Yes", + ), + ( + "7e81f94e-6441-4430-a12d-76df09181b66", + "97be337c-ff27-4869-bf63-ef1abc9df15d", + "Yes", + ), + ( + "390d6507-5029-4dae-bcd4-ce7178c9b560", + "34944d4f-762e-4262-8c79-b9fd48521ca0", + "Yes", + ), + ( + "97a5ddc0-d4e0-43ac-a571-9722405a0a9b", + "3e8c0c39-3f30-4c9b-a449-85eef1b2a458", + "Yes", + ), + ], + }, + { + "value": "63767e4b-9ce8-4fe2-8724-65cc1f763de0", + "label": "No", + "applies_to": [ + ( + "856d2d65-cd25-49fa-8da9-cabb78292894", + "63767e4b-9ce8-4fe2-8724-65cc1f763de0", + "No", + ), + ( + "1dad74a2-95df-4825-bbba-dca8b91d2371", + "697c0883-798d-4af7-b8b6-101c7f709cd5", + "No", + ), + ( + "7e81f94e-6441-4430-a12d-76df09181b66", + "77355172-b437-4324-9dcc-e2607ad27cb1", + "No", + ), + ( + "390d6507-5029-4dae-bcd4-ce7178c9b560", + "63be6081-bee8-4cf5-a453-91893e31940f", + "No", + ), + ( + "97a5ddc0-d4e0-43ac-a571-9722405a0a9b", + "7f5244fe-590b-4e38-beaf-0cf1ccb9e71b", + "No", + ), + ], + }, + ], + } + ] def test_processing_configuration_file_exists_with_None(): diff --git a/src/dashboard/src/components/administration/forms.py b/src/dashboard/src/components/administration/forms.py index 62a4d8a6a9..bb029aa8d5 100644 --- a/src/dashboard/src/components/administration/forms.py +++ b/src/dashboard/src/components/administration/forms.py @@ -30,8 +30,6 @@ from installer.forms import site_url_field, load_site_url from main.models import Agent, TaxonomyTerm -import storageService as storage_service - class AgentForm(forms.ModelForm): class Meta: @@ -260,57 +258,21 @@ class Meta: class ProcessingConfigurationForm(forms.Form): - """ - Build processing configuration form bounded to a processingMCP document. + """Processing configuration form bound to the processing configuration + fields dictated by MCPServer. Field values are populated after the contents + of the processing configuration document. - Every processing field in this form requires the following - properties: type, name, label. In addition, these are some other - constraints based on the type: + TODO: certain responsibilites should be handled by MCPServer, namely: + (1) load pre-configured choices, and (2) additional RPC to save the + configuration. """ - EMPTY_OPTION_NAME = _("None") - EMPTY_CHOICES = [(None, EMPTY_OPTION_NAME)] + EMPTY_CHOICES = [(None, _("None"))] DEFAULT_FIELD_OPTS = {"required": False, "initial": None} - # This extends the fields defined in the processing_config module. - # Temporary solution until workflow data can be translated. LABELS = { - "bd899573-694e-4d33-8c9b-df0af802437d": _("Assign UUIDs to directories"), - "56eebd45-5600-4768-a8c2-ec0114555a3d": _("Generate transfer structure report"), - "f09847c2-ee51-429a-9478-a860477f6b8d": _( - "Perform file format identification (Transfer)" - ), - "dec97e3c-5598-4b99-b26e-f87a435a6b7f": _("Extract packages"), - "f19926dd-8fb5-4c79-8ade-c83f61f55b40": _("Delete packages after extraction"), - "70fc7040-d4fb-4d19-a0e6-792387ca1006": _("Perform policy checks on originals"), - "accea2bf-ba74-4a3a-bb97-614775c74459": _("Examine contents"), - "bb194013-597c-4e4a-8493-b36d190f8717": _("Create SIP(s)"), - "7a024896-c4f7-4808-a240-44c87c762bc5": _( - "Perform file format identification (Ingest)" - ), - "cb8e5706-e73f-472f-ad9b-d1236af8095f": _("Normalize"), - "de909a42-c5b5-46e1-9985-c031b50e9d30": _("Approve normalization"), - "498f7a6d-1b8c-431a-aa5d-83f14f3c5e65": _("Generate thumbnails"), - "153c5f41-3cfb-47ba-9150-2dd44ebc27df": _( - "Perform policy checks on preservation derivatives" - ), - "8ce07e94-6130-4987-96f0-2399ad45c5c2": _( - "Perform policy checks on access derivatives" - ), - "a2ba5278-459a-4638-92d9-38eb1588717d": _("Bind PIDs"), - "d0dfa5fc-e3c2-4638-9eda-f96eea1070e0": _("Document empty directories"), - "eeb23509-57e2-4529-8857-9d62525db048": _("Reminder: add metadata if desired"), - "82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6": _("Transcribe files (OCR)"), - "087d27be-c719-47d8-9bbb-9a7d8b609c44": _( - "Perform file format identification (Submission documentation & metadata)" - ), - "01d64f58-8295-4b7b-9cab-8f1b153a504f": _("Select compression algorithm"), - "01c651cb-c174-4ba4-b985-1d87a44d6754": _("Select compression level"), - "2d32235c-02d4-4686-88a6-96f4d6c7b1c3": _("Store AIP"), - "b320ce81-9982-408a-9502-097d0daa48fa": _("Store AIP location"), - "92879a29-45bf-4f0b-ac43-e64474f0f2f9": _("Upload DIP"), - "5e58066d-e113-4383-b20b-f301ed4d751c": _("Store DIP"), - "cd844b6e-ab3c-4bc6-b34f-7103f88715de": _("Store DIP location"), + # MCPServer does not extract messages. + "virus_scanning": _("Virus scanning") } NAME_MAX_LENGTH = 50 @@ -331,129 +293,89 @@ class ProcessingConfigurationForm(forms.Form): def __init__(self, *args, **kwargs): user = kwargs.pop("user") super(ProcessingConfigurationForm, self).__init__(*args, **kwargs) - self._load_processing_config_fields(user) - for choice_uuid, field in self.processing_fields.items(): - ftype = field["type"] + self.load_processing_config_fields(user) + self.create_fields() + + def load_processing_config_fields(self, user): + """Obtain processing fields and available choices from MCPServer.""" + client = MCPClient(user) + self.processing_fields = client.get_processing_config_fields() + + # Override labels with translations in this form. + for field in self.processing_fields: + try: + name = field["name"] + field["label"] = self.LABELS[name] + except KeyError: + pass + + def create_fields(self): + """Set up form fields with the server data.""" + for field in self.processing_fields: opts = self.DEFAULT_FIELD_OPTS.copy() opts["label"] = field["label"] - choices = opts["choices"] = list(self.EMPTY_CHOICES) - if ftype == "boolean": - if "yes_option" in field: - choices.append((field["yes_option"], _("Yes"))) - if "no_option" in field: - choices.append((field["no_option"], _("No"))) - elif ftype in ("chain_choice", "replace_dict"): - choices.extend(field["options"]) - elif ftype == "storage_service": - choices.append( - ( - "/api/v2/location/default/{}/".format(field["purpose"]), - _("Default location"), - ) - ) - for loc in get_storage_locations(purpose=field["purpose"]): - label = loc["description"] - if not label: - label = loc["relative_path"] - choices.append((loc["resource_uri"], label)) - self.fields[choice_uuid] = forms.ChoiceField( + opts["choices"] = self.EMPTY_CHOICES[:] + for item in field["choices"]: + opts["choices"].append((item["value"], item["label"])) + self.fields[field["id"]] = forms.ChoiceField( widget=Select(attrs={"class": "form-control"}), **opts ) - def _load_processing_config_fields(self, user): - client = MCPClient(user) - self.processing_fields = client.get_processing_config_fields() - for choice_uuid, field in self.processing_fields.items(): - field["label"] = self.LABELS[choice_uuid] + def get_processing_config_path(self, name): + return os.path.join( + helpers.processing_config_path(), "{}ProcessingMCP.xml".format(name) + ) def load_config(self, name): - """ - Bound the choices found in the XML document to the form fields. - """ + """Populate form fields with the choices found in the XML document.""" self.fields["name"].initial = name self.fields["name"].widget.attrs["readonly"] = "readonly" - config_path = os.path.join( - helpers.processing_config_path(), "{}ProcessingMCP.xml".format(name) - ) + + # Build a map with all pre-configuired choices for quick access. + config_path = self.get_processing_config_path(name) root = etree.parse(config_path) - for choice in root.findall(".//preconfiguredChoice"): - applies_to = choice.findtext("appliesTo") - go_to_chain = choice.findtext("goToChain") - fprops = self.processing_fields.get(applies_to) - field = self.fields.get(applies_to) - if fprops is None or go_to_chain is None or field is None: + choices = { + choice.findtext("appliesTo"): choice.findtext("goToChain") + for choice in root.findall(".//preconfiguredChoice") + } + + for processing_field in self.processing_fields: + link_id = processing_field["id"] + form_field = self.fields[link_id] + try: + value = choices[link_id] + except KeyError: continue - field.initial = go_to_chain + form_field.initial = value def save_config(self): - """ - Encode the configuration to XML and write it to disk. - """ + """Encode the configuration to XML and write it to disk.""" + # Capture the config name, then discard the form field. name = self.cleaned_data["name"] del self.cleaned_data["name"] - config_path = os.path.join( - helpers.processing_config_path(), "{}ProcessingMCP.xml".format(name) - ) - config = PreconfiguredChoices() - for choice_uuid, value in self.cleaned_data.items(): - fprops = self.processing_fields.get(choice_uuid) - if fprops is None or value is None: - continue - field = self.fields.get(choice_uuid) - if field is None: - continue - if isinstance(field, forms.ChoiceField): - if not value: # Ignore empty string! - continue - if fprops["type"] == "days": - if value == 0: - continue - delay = str(float(value) * (24 * 60 * 60)) - config.add_choice( - choice_uuid, - fprops["chain"], - delay_duration=delay, - comment=fprops["label"], + + config = PreconfiguredChoices() # Configuration encoder. + for field in self.processing_fields: + link_id = field["id"] + try: + choice = self.cleaned_data[link_id] + except KeyError: + raise forms.ValidationError("Unknown processing field %s" % link_id) + if not choice: + continue # Ignore when user chose None. + matches = None + for item in field["choices"]: + if item["value"] == choice: + matches = item["applies_to"] + if not matches: + raise forms.ValidationError( + "Unknown value for processing field %s: %s", link_id, choice ) - elif fprops["type"] == "chain_choice" and "duplicates" in fprops: - # If we have more than one chain (duplicates) then we need to - # add them all, e.g.:: - # - # - # - # ... - # ... - # - # - # - # ... - # ... - # - # - # Otherwise, when `KeyError` is raised, add single entry. - try: - desc, matches = fprops["duplicates"][value] - except KeyError: - config.add_choice(choice_uuid, value, comment=fprops["label"]) - else: - for i, match in enumerate(matches): - comment = '{} (match {} for "{}")'.format( - fprops["label"], i + 1, desc - ) - config.add_choice(match[0], match[1], comment=comment) - else: - config.add_choice(choice_uuid, value, comment=fprops["label"]) - config.save(config_path) - - -def get_storage_locations(purpose): - try: - dirs = storage_service.get_location(purpose=purpose) - if len(dirs) == 0: - raise Exception("Storage server improperly configured.") - except Exception: - dirs = [] - return dirs + for applies_to, go_to_chain, label in matches: + comment = "{}: {}".format(self.fields[link_id].label, label) + config.add_choice(applies_to, go_to_chain, comment) + + config.save(self.get_processing_config_path(name)) class PreconfiguredChoices(object): @@ -465,19 +387,13 @@ def __init__(self): self.xml = etree.Element("processingMCP") self.choices = etree.SubElement(self.xml, "preconfiguredChoices") - def add_choice( - self, applies_to_text, go_to_chain_text, delay_duration=None, comment=None - ): + def add_choice(self, applies_to_text, go_to_chain_text, comment=None): if comment is not None: comment = etree.Comment(" {} ".format(comment)) self.choices.append(comment) choice = etree.SubElement(self.choices, "preconfiguredChoice") etree.SubElement(choice, "appliesTo").text = applies_to_text etree.SubElement(choice, "goToChain").text = go_to_chain_text - if delay_duration is not None: - etree.SubElement( - choice, "delay", {"unitCtime": "yes"} - ).text = delay_duration def save(self, config_path): with open(config_path, "w") as f: diff --git a/src/dashboard/src/contrib/mcp/client.py b/src/dashboard/src/contrib/mcp/client.py index c937358238..598074d487 100644 --- a/src/dashboard/src/contrib/mcp/client.py +++ b/src/dashboard/src/contrib/mcp/client.py @@ -203,7 +203,8 @@ def approve_partial_reingest(self, sip_uuid): return self._rpc_sync_call("approvePartialReingest", data) def get_processing_config_fields(self): - return self._rpc_sync_call("getProcessingConfigFields") + data = {"lang": self.lang} + return self._rpc_sync_call("getProcessingConfigFields", data) def _get_units_statuses(self, type_): data = {"type": type_, "lang": self.lang}