diff --git a/pyproject.toml b/pyproject.toml index d0f495b..236730b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,11 @@ python = "^3.9" httpx = {version = "0.*", optional = true} msgspec = "0.*" lxml = {version = "5.*", optional = true} +xmltodict = {version = "0.*", optional = true} [tool.poetry.extras] fmr = ["httpx"] -xml = ["lxml"] +xml = ["lxml", "xmltodict"] [tool.poetry.group.dev.dependencies] darglint = "^1.8.1" @@ -47,6 +48,8 @@ pytest-cov = "^4.0.0" respx = "^0.20.2" pyroma = "^4.2" lxml-stubs = "^0.5.1" +types-xmltodict = "^0.13.0.3" + [tool.poetry.group.docs.dependencies] sphinx = "^7.2.6" diff --git a/src/pysdmx/io/input_processor.py b/src/pysdmx/io/input_processor.py index c4def93..61f2983 100644 --- a/src/pysdmx/io/input_processor.py +++ b/src/pysdmx/io/input_processor.py @@ -6,8 +6,6 @@ from pathlib import Path from typing import Tuple, Union -from pysdmx.io.xml.sdmx21.doc_validation import validate_doc - def __remove_bom(input_string: str) -> str: return input_string.replace("\ufeff", "") @@ -15,7 +13,6 @@ def __remove_bom(input_string: str) -> str: def __check_xml(infile: str) -> bool: if infile[:5] == " Dict[str, Any]: + """Reads an SDMX-ML file and returns a dictionary with the parsed data. + + Args: + infile: Path to file, URL, or string. + validate: If True, the XML data will be validated against the XSD. + mode: The type of message to parse. + use_dataset_id: If True, the dataset ID will be used as the key in the + resulting dictionary. + + Returns: + dict: Dictionary with the parsed data. + + Raises: + ValueError: If the SDMX data cannot be parsed. + """ + if validate: + validate_doc(infile) + dict_info = xmltodict.parse( + infile, **XML_OPTIONS # type: ignore[arg-type] + ) + + del infile + + if mode is not None and MODES[mode.value] not in dict_info: + raise ValueError( + f"Unable to parse sdmx file as {MODES[mode.value]} file" + ) + + result = __generate_sdmx_objects_from_xml(dict_info, use_dataset_id) + + return result + + +def __generate_sdmx_objects_from_xml( + dict_info: Dict[str, Any], use_dataset_id: bool = False +) -> Dict[str, Any]: + """Generates SDMX objects from the XML dictionary (xmltodict). + + Args: + dict_info: XML dictionary (xmltodict) + use_dataset_id: Use the dataset ID as the key in + the resulting dictionary + + Returns: + dict: Dictionary with the parsed data. + + Raises: + ClientError: If a SOAP error message is found. + ValueError: If the SDMX data cannot be parsed. + """ + if ERROR in dict_info: + code = dict_info[ERROR][ERROR_MESSAGE][ERROR_CODE] + text = dict_info[ERROR][ERROR_MESSAGE][ERROR_TEXT] + raise ClientError(int(code), text) + # Leaving this commented for metadata read (#39) + # if STRUCTURE in dict_info: + # return create_structures(dict_info[STRUCTURE][STRUCTURES]) + if REG_INTERFACE in dict_info: + return handle_registry_interface(dict_info) + raise ValueError("Cannot parse this sdmx data") diff --git a/src/pysdmx/io/xml/sdmx21/reader/submission_reader.py b/src/pysdmx/io/xml/sdmx21/reader/submission_reader.py new file mode 100644 index 0000000..c802e26 --- /dev/null +++ b/src/pysdmx/io/xml/sdmx21/reader/submission_reader.py @@ -0,0 +1,39 @@ +"""Read SDMX-ML submission messages.""" + +from typing import Any, Dict + +from pysdmx.io.xml.sdmx21.__parsing_config import ( + ACTION, + MAINTAINABLE_OBJECT, + REG_INTERFACE, + STATUS, + STATUS_MSG, + SUBMISSION_RESULT, + SUBMIT_STRUCTURE_RESPONSE, + SUBMITTED_STRUCTURE, + URN, +) +from pysdmx.model.submission import SubmissionResult +from pysdmx.util import parse_urn + + +def handle_registry_interface(dict_info: Dict[str, Any]) -> Dict[str, Any]: + """Handle the Registry Interface message. + + Args: + dict_info: Dictionary with the parsed data. + + Returns: + dict: Dictionary with the parsed data. + """ + response = dict_info[REG_INTERFACE][SUBMIT_STRUCTURE_RESPONSE] + + result = {} + for submission_result in response[SUBMISSION_RESULT]: + structure = submission_result[SUBMITTED_STRUCTURE] + action = structure[ACTION] + urn = structure[MAINTAINABLE_OBJECT][URN] + full_id = parse_urn(urn).full_id + status = submission_result[STATUS_MSG][STATUS] + result[full_id] = SubmissionResult(action, full_id, status) + return result diff --git a/src/pysdmx/model/submission.py b/src/pysdmx/model/submission.py new file mode 100644 index 0000000..3173ed9 --- /dev/null +++ b/src/pysdmx/model/submission.py @@ -0,0 +1,20 @@ +"""SDMX Submission classes.""" + +from msgspec import Struct + + +class SubmissionResult(Struct): + """A class to represent a Submission Result.""" + + action: str + full_id: str + status: str + + def __str__(self) -> str: + """Return a string representation of the SubmissionResult.""" + return ( + f"" + ) diff --git a/src/pysdmx/util/__init__.py b/src/pysdmx/util/__init__.py index 732645d..87652a9 100644 --- a/src/pysdmx/util/__init__.py +++ b/src/pysdmx/util/__init__.py @@ -25,6 +25,11 @@ class Reference(Struct, frozen=True): id: str version: str + @property + def full_id(self) -> str: + """Returns the full ID of the referenced artefact.""" + return f"{self.agency}:{self.id}({self.version})" + class ItemReference(Struct, frozen=True): """The coordinates of an SDMX non-nested item. diff --git a/tests/io/test_input_processor.py b/tests/io/test_input_processor.py index ab5a358..d8ed27d 100644 --- a/tests/io/test_input_processor.py +++ b/tests/io/test_input_processor.py @@ -4,6 +4,7 @@ import pytest from pysdmx.io.input_processor import process_string_to_read +from pysdmx.io.xml.sdmx21.reader import read_xml @pytest.fixture() @@ -70,8 +71,9 @@ def test_process_string_to_read_bom(valid_xml, valid_xml_bom): def test_process_string_to_read_invalid_xml(invalid_xml): message = "This element is not expected." + process_string_to_read(invalid_xml) with pytest.raises(ValueError, match=message): - process_string_to_read(invalid_xml) + read_xml(invalid_xml, validate=True) def test_process_string_to_read_invalid_type(): @@ -101,4 +103,5 @@ def test_process_string_to_read_invalid_allowed_error(invalid_allowed_error): # This is a valid XML file, but it contains an error that is allowed # QName value on xsi:type attribute does not resolve to a type definition infile, filetype = process_string_to_read(invalid_allowed_error) - assert filetype == "xml" + with pytest.raises(ValueError, match="Cannot parse this sdmx data"): + read_xml(infile, validate=True) diff --git a/tests/io/xml/sdmx21/reader/samples/error_304.xml b/tests/io/xml/sdmx21/reader/samples/error_304.xml new file mode 100644 index 0000000..1d83d5d --- /dev/null +++ b/tests/io/xml/sdmx21/reader/samples/error_304.xml @@ -0,0 +1,12 @@ + + + + + Either no structures were submitted, + or the submitted structures contain no changes from the ones + currently stored in the system + + + \ No newline at end of file diff --git a/tests/io/xml/sdmx21/reader/samples/submission_append.xml b/tests/io/xml/sdmx21/reader/samples/submission_append.xml new file mode 100644 index 0000000..1d3f451 --- /dev/null +++ b/tests/io/xml/sdmx21/reader/samples/submission_append.xml @@ -0,0 +1,32 @@ + + + + test + true + 2023-11-08T12:40:53Z + + + + + + + + urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=BIS:BIS_DER(1.0) + + + + + + + + urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS:WEBSTATS_DER_DATAFLOW(1.0) + + + + + + + + \ No newline at end of file diff --git a/tests/io/xml/sdmx21/reader/test_reader.py b/tests/io/xml/sdmx21/reader/test_reader.py new file mode 100644 index 0000000..10dd606 --- /dev/null +++ b/tests/io/xml/sdmx21/reader/test_reader.py @@ -0,0 +1,68 @@ +from pathlib import Path + +import pytest + +from pysdmx.errors import ClientError +from pysdmx.io.input_processor import process_string_to_read +from pysdmx.io.xml.enums import MessageType +from pysdmx.io.xml.sdmx21.reader import read_xml +from pysdmx.model.submission import SubmissionResult + + +# Test parsing SDMX Registry Interface Submission Response + + +@pytest.fixture() +def submission_path(): + return Path(__file__).parent / "samples" / "submission_append.xml" + + +@pytest.fixture() +def error_304_path(): + return Path(__file__).parent / "samples" / "error_304.xml" + + +def test_submission_result(submission_path): + input_str, filetype = process_string_to_read(submission_path) + assert filetype == "xml" + result = read_xml(input_str, validate=True) + + full_id_1 = "BIS:BIS_DER(1.0)" + full_id_2 = "BIS:WEBSTATS_DER_DATAFLOW(1.0)" + + assert full_id_1 in result + submission_1 = result[full_id_1] + assert isinstance(submission_1, SubmissionResult) + assert submission_1.action == "Append" + assert submission_1.full_id == full_id_1 + assert submission_1.status == "Success" + + assert full_id_2 in result + submission_2 = result[full_id_2] + assert isinstance(submission_2, SubmissionResult) + assert submission_2.action == "Append" + assert submission_2.full_id == full_id_2 + assert submission_2.status == "Success" + + +def test_error_304(error_304_path): + input_str, filetype = process_string_to_read(error_304_path) + assert filetype == "xml" + with pytest.raises(ClientError) as e: + read_xml(input_str, validate=False, mode=MessageType.Error) + assert e.value.status == 304 + reference_title = ( + "Either no structures were submitted,\n" + " or the submitted structures " + "contain no changes from the ones\n" + " currently stored in the system" + ) + + assert e.value.title == reference_title + + +def test_error_message_with_different_mode(error_304_path): + input_str, filetype = process_string_to_read(error_304_path) + assert filetype == "xml" + with pytest.raises(ValueError, match="Unable to parse sdmx file as"): + read_xml(input_str, validate=True, mode=MessageType.Submission) diff --git a/tests/model/test_submission.py b/tests/model/test_submission.py new file mode 100644 index 0000000..f4f01ec --- /dev/null +++ b/tests/model/test_submission.py @@ -0,0 +1,31 @@ +import pytest + +from pysdmx.model.submission import SubmissionResult + + +@pytest.fixture() +def action(): + return "Append" + + +@pytest.fixture() +def full_id(): + return "BIS:BIS_DER(1.0)" + + +@pytest.fixture() +def status(): + return "Success" + + +def test_full_instantiation(action, full_id, status): + submission_result = SubmissionResult(action, full_id, status) + + assert submission_result.action == action + assert submission_result.full_id == full_id + assert submission_result.status == status + assert str(submission_result) == ( + f"" + )