From c9a7aed526a89e4a9240910eb2aae353c3411c9f Mon Sep 17 00:00:00 2001 From: "javier.hernandez" Date: Tue, 14 May 2024 22:08:34 +0200 Subject: [PATCH 1/9] Initial writing methods for metadata in SDMX-ML. Added enums for Message Type and Action. Signed-off-by: javier.hernandez --- src/pysdmx/model/message.py | 32 ++++ src/pysdmx/writers/__init__.py | 1 + src/pysdmx/writers/__write_aux.py | 289 ++++++++++++++++++++++++++++++ src/pysdmx/writers/write.py | 47 +++++ 4 files changed, 369 insertions(+) create mode 100644 src/pysdmx/model/message.py create mode 100644 src/pysdmx/writers/__init__.py create mode 100644 src/pysdmx/writers/__write_aux.py create mode 100644 src/pysdmx/writers/write.py diff --git a/src/pysdmx/model/message.py b/src/pysdmx/model/message.py new file mode 100644 index 0000000..e6bbf10 --- /dev/null +++ b/src/pysdmx/model/message.py @@ -0,0 +1,32 @@ +"""Message module. + +This module contains the enumeration for the different types of messages that +can be written. +""" + +from enum import Enum + + +class MessageType(Enum): + """MessageType enumeration. + + Enumeration that withholds the Message type for writing purposes. + """ + + GenericDataSet = 1 + StructureSpecificDataSet = 2 + Metadata = 3 + Error = 4 + Submission = 5 + + +class ActionType(Enum): + """ActionType enumeration. + + Enumeration that withholds the Action type for writing purposes. + """ + + Append = "append" + Replace = "replace" + Delete = "delete" + Information = "information" diff --git a/src/pysdmx/writers/__init__.py b/src/pysdmx/writers/__init__.py new file mode 100644 index 0000000..fec157a --- /dev/null +++ b/src/pysdmx/writers/__init__.py @@ -0,0 +1 @@ +"""Writer module.""" diff --git a/src/pysdmx/writers/__write_aux.py b/src/pysdmx/writers/__write_aux.py new file mode 100644 index 0000000..cbcafe7 --- /dev/null +++ b/src/pysdmx/writers/__write_aux.py @@ -0,0 +1,289 @@ +"""Writer auxiliary functions.""" + +from datetime import datetime +from typing import Any, Dict + +from pysdmx.model.message import ActionType, MessageType + +MESSAGE_TYPE_MAPPING = { + MessageType.GenericDataSet: "GenericData", + MessageType.StructureSpecificDataSet: "StructureSpecificData", + MessageType.Metadata: "Structure", +} + +ABBR_MSG = "mes" +ABBR_GEN = "gen" +ABBR_COM = "com" +ABBR_STR = "str" +ABBR_SPE = "ss" + +ANNOTATIONS = "Annotations" +STRUCTURES = "Structures" +ORGS = "OrganisationSchemes" +AGENCIES = "AgencyScheme" +CODELISTS = "Codelists" +CONCEPTS = "Concepts" +DSDS = "DataStructures" +DATAFLOWS = "Dataflows" +CONSTRAINTS = "Constraints" + +BASE_URL = "http://www.sdmx.org/resources/sdmxml/schemas/v2_1" + +NAMESPACES = { + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + ABBR_MSG: f"{BASE_URL}/message", + ABBR_GEN: f"{BASE_URL}/generic", + ABBR_COM: f"{BASE_URL}/common", + ABBR_STR: f"{BASE_URL}/structure", + ABBR_SPE: f"{BASE_URL}/structureSpecific", +} + +URN_DS_BASE = "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=" + + +def __namespaces_from_type(type_: MessageType) -> str: + """Returns the namespaces for the XML file based on type. + + Args: + type_: MessageType to be used + + Returns: + A string with the namespaces + """ + if type_ == MessageType.GenericDataSet: + return f"xmlns:{ABBR_GEN}={NAMESPACES[ABBR_GEN]!r} " + elif type_ == MessageType.StructureSpecificDataSet: + return f"xmlns:{ABBR_SPE}={NAMESPACES[ABBR_SPE]!r} " + elif type_ == MessageType.Metadata: + return f"xmlns:{ABBR_STR}={NAMESPACES[ABBR_STR]!r} " + else: + return "" + + +def __namespaces_from_content(content: Dict[str, Any]) -> str: + """Returns the namespaces for the XML file based on content. + + Args: + content: Datasets or None + + Returns: + A string with the namespaces + + Raises: + Exception: If the dataset has no structure defined + """ + outfile = "" + for i, key in enumerate(content): + if content[key].structure is None: + raise Exception(f"Dataset {key} has no structure defined") + ds_urn = URN_DS_BASE + ds_urn += ( + f"{content[key].structure.unique_id}:" + f"ObsLevelDim:{content[key].dim_at_obs}" + ) + outfile += f"xmlns:ns{i}={ds_urn!r}" + return outfile + + +def create_namespaces( + type_: MessageType, content: Dict[str, Any], prettyprint: bool = False +) -> str: + """Creates the namespaces for the XML file. + + Args: + type_: MessageType to be used + content: Datasets or None + prettyprint: Prettyprint or not + + Returns: + A string with the namespaces + """ + nl = "\n" if prettyprint else "" + + outfile = f'{nl}' + + outfile += f"<{ABBR_MSG}:{MESSAGE_TYPE_MAPPING[type_]} " + outfile += f'xmlns:xsi={NAMESPACES["xsi"]!r} ' + outfile += f"xmlns:{ABBR_MSG}={NAMESPACES[ABBR_MSG]!r} " + outfile += __namespaces_from_type(type_) + if type_ == MessageType.StructureSpecificDataSet: + outfile += __namespaces_from_content(content) + outfile += ( + f"xmlns:{ABBR_COM}={NAMESPACES[ABBR_COM]!r} " + f'xsi:schemaLocation="{NAMESPACES[ABBR_MSG]} ' + f'https://registry.sdmx.org/schemas/v2_1/SDMXMessage.xsd">' + ) + + return outfile + + +DEFAULT_HEADER = { + "ID": "test", + "Test": "true", + "Prepared": datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + "Sender": "Unknown", + "Receiver": "Not_Supplied", + "DataSetAction": ActionType.Information.value, + "Source": "PySDMX", +} + + +def __generate_value_element(element: str, prettyprint: bool) -> str: + """Generates a value element for the XML file (XML tag with value). + + Args: + element: ID, Test, Prepared, Sender, Receiver, DataSetAction, Source + prettyprint: Prettyprint or not + + Returns: + A string with the value element + """ + nl = "\n" if prettyprint else "" + child2 = "\t\t" if prettyprint else "" + return ( + f"{nl}{child2}<{ABBR_MSG}:{element}>" + f"{DEFAULT_HEADER[element]}" + f"" + ) + + +def __generate_item_element(element: str, prettyprint: bool) -> str: + """Generates an item element for the XML file (XML tag with id attribute). + + Args: + element: Sender, Receiver + prettyprint: Prettyprint or not + + Returns: + A string with the item element + """ + nl = "\n" if prettyprint else "" + child2 = "\t\t" if prettyprint else "" + return ( + f"{nl}{child2}<{ABBR_MSG}:{element} id={DEFAULT_HEADER[element]!r}/>" + ) + + +def __generate_structure_element( + content: Dict[str, Any], prettyprint: bool +) -> str: + return "" + + +def generate_new_header( + type_: MessageType, datasets: Dict[str, Any], prettyprint: bool +) -> str: + """Writes the header to the XML file. + + Args: + type_: MessageType to be used + datasets: Datasets or None + prettyprint: Prettyprint or not + + Returns: + A string with the header + + Raises: + NotImplementedError: If the MessageType is not Metadata + """ + if type_ != MessageType.Metadata: + raise NotImplementedError("Only Metadata messages are supported") + + nl = "\n" if prettyprint else "" + child1 = "\t" if prettyprint else "" + + outfile = f"{nl}{child1}<{ABBR_MSG}:Header>" + outfile += __generate_value_element("ID", prettyprint) + outfile += __generate_value_element("Test", prettyprint) + outfile += __generate_value_element("Prepared", prettyprint) + outfile += __generate_item_element("Sender", prettyprint) + outfile += __generate_item_element("Receiver", prettyprint) + if type_.value < MessageType.Metadata.value: + outfile += __generate_structure_element(datasets, prettyprint) + outfile += __generate_value_element("DataSetAction", prettyprint) + outfile += __generate_value_element("Source", prettyprint) + outfile += f"{nl}{child1}" + return outfile + + +def __write_metadata_element( + package: Dict[str, Any], key: str, prettyprint: object +) -> str: + """Writes the metadata element to the XML file. + + Args: + package: The package to be written + key: The key to be used + prettyprint: Prettyprint or not + + Returns: + A string with the metadata element + """ + outfile = "" + nl = "\n" if prettyprint else "" + child2 = "\t\t" if prettyprint else "" + + if key in package: + outfile += f"{nl}{child2}<{ABBR_STR}:{MSG_CONTENT_PKG[key]}>" + # TODO: Add the __to_XML method to the Item and ItemScheme classes + # for item in package[key].values(): + # outfile += item.__to_XML(MSG_CONTENT_ITEM[key], prettyprint) + outfile += f"{nl}{child2}" + + return outfile + + +MSG_CONTENT_PKG = { + ORGS: "OrganisationSchemes", + DATAFLOWS: "Dataflows", + CODELISTS: "Codelists", + CONCEPTS: "Concepts", + DSDS: "DataStructures", + CONSTRAINTS: "Constraints", +} + +MSG_CONTENT_ITEM = { + ORGS: "AgencyScheme", + DATAFLOWS: "Dataflow", + CODELISTS: "Codelist", + CONCEPTS: "ConceptScheme", + DSDS: "DataStructure", + CONSTRAINTS: "ContentConstraint", +} + + +def generate_structures(content: Dict[str, Any], prettyprint: bool) -> str: + """Writes the structures to the XML file. + + Args: + content: The Message Content to be written + prettyprint: Prettyprint or not + + Returns: + A string with the structures + """ + nl = "\n" if prettyprint else "" + child1 = "\t" if prettyprint else "" + + outfile = f"{nl}{child1}<{ABBR_MSG}:Structures>" + + for key in MSG_CONTENT_PKG: + outfile += __write_metadata_element(content, key, prettyprint) + + outfile += f"{nl}{child1}" + + return outfile + + +def get_end_message(type_: MessageType, prettyprint: bool) -> str: + """Returns the end message for the XML file. + + Args: + type_: MessageType to be used + prettyprint: Prettyprint or not + + Returns: + A string with the end message + """ + nl = "\n" if prettyprint else "" + return f"{nl}" diff --git a/src/pysdmx/writers/write.py b/src/pysdmx/writers/write.py new file mode 100644 index 0000000..b0cc80e --- /dev/null +++ b/src/pysdmx/writers/write.py @@ -0,0 +1,47 @@ +"""Writing SDMX-ML files from Message content.""" + +from typing import Any, Dict, Optional + +from pysdmx.model.message import MessageType +from pysdmx.writers.__write_aux import ( + create_namespaces, + generate_new_header, + generate_structures, + get_end_message, +) + + +def writer( + content: Dict[str, Any], + type_: MessageType, + path: str = "", + prettyprint: bool = True, +) -> Optional[str]: + """This function writes a SDMX-ML file from the Message Content. + + Args: + content: The content to be written + type_: The type of message to be written + path: The path to save the file + prettyprint: Prettyprint or not + + Returns: + The XML string if path is empty, None otherwise + + """ + outfile = create_namespaces(type_, content, prettyprint) + + outfile += generate_new_header(type_, content, prettyprint) + + if type_ == MessageType.Metadata: + outfile += generate_structures(content, prettyprint) + + outfile += get_end_message(type_, prettyprint) + + if path == "": + return outfile + + with open(path, "w", encoding="UTF-8", errors="replace") as f: + f.write(outfile) + + return None From 3da08ac33b796483254c49cb60e02ac66f7a7612 Mon Sep 17 00:00:00 2001 From: "javier.hernandez" Date: Wed, 15 May 2024 00:23:05 +0200 Subject: [PATCH 2/9] Added _to_XML methods to Item and ItemScheme. Added also these methods to superclasses. Added auxiliary methods to writer_aux. Signed-off-by: javier.hernandez --- src/pysdmx/model/__base.py | 182 +++++++++++++++++++++++++++++- src/pysdmx/writers/__write_aux.py | 71 ++++++++++-- src/pysdmx/writers/write.py | 1 + 3 files changed, 242 insertions(+), 12 deletions(-) diff --git a/src/pysdmx/model/__base.py b/src/pysdmx/model/__base.py index 69d97e5..2ad450a 100644 --- a/src/pysdmx/model/__base.py +++ b/src/pysdmx/model/__base.py @@ -1,9 +1,15 @@ from datetime import datetime -from typing import Any, Dict, Optional, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Union from msgspec import Struct from pysdmx.errors import ClientError +from pysdmx.writers.__write_aux import ( + ABBR_COM, + ABBR_STR, + add_indent, + export_intern_data, +) class Annotation(Struct, frozen=True, omit_defaults=True): @@ -57,6 +63,14 @@ def __str__(self) -> str: return ", ".join(out) +ANNOTATION_WRITER = { + "title": "AnnotationTitle", + "type": "AnnotationType", + "url": "AnnotationURL", + "text": "AnnotationText", +} + + class AnnotableArtefact(Struct, frozen=True, omit_defaults=True, kw_only=True): """Annotable Artefact class. @@ -86,6 +100,37 @@ def __str__(self) -> str: out.append(f"{k}={str(v)}") return ", ".join(out) + def _to_XML(self, indent: str) -> Any: + + if len(self.annotations) == 0: + return "" + + child1 = indent + child2 = add_indent(child1) + + outfile = f"<{ABBR_COM}:Annotations>" + for annotation in self.annotations: + if annotation.id is None: + outfile += f"{child1}<{ABBR_COM}:Annotation>" + else: + outfile += ( + f"{child1}<{ABBR_COM}:Annotation " f"id={annotation.id!r}>" + ) + + for attr, label in ANNOTATION_WRITER.items(): + if getattr(annotation, attr, None) is not None: + value = getattr(annotation, attr) + value = value.replace("&", "&").rstrip() + outfile += ( + f"{child2}<{ABBR_COM}:{label}>" + f"{value}" + f"" + ) + + outfile += f"{child1}" + outfile += f"" + return outfile + class IdentifiableArtefact(AnnotableArtefact, frozen=True, omit_defaults=True): """Identifiable Artefact class. @@ -103,6 +148,25 @@ class IdentifiableArtefact(AnnotableArtefact, frozen=True, omit_defaults=True): uri: Optional[str] = None urn: Optional[str] = None + def _to_XML(self, indent: str) -> Dict[str, Any]: + attributes = "" + + if self.id is not None: + attributes += f" id={self.id!r}" + + if self.uri is not None: + attributes += f" uri={self.uri!r}" + + if self.urn is not None: + attributes += f" urn={self.urn!r}" + + outfile = { + "Annotations": super(IdentifiableArtefact, self)._to_XML(indent), + "Attributes": attributes, + } + + return outfile + class NameableArtefact(IdentifiableArtefact, frozen=True, omit_defaults=True): """Nameable Artefact class. @@ -117,6 +181,31 @@ class NameableArtefact(IdentifiableArtefact, frozen=True, omit_defaults=True): name: Optional[str] = None description: Optional[str] = None + def _to_XML(self, indent: str) -> Dict[str, Any]: + outfile = super(NameableArtefact, self)._to_XML(indent) + + if self.name is not None: + outfile["Name"] = [ + ( + f"{add_indent(indent)}" + f'<{ABBR_COM}:Name xml:lang="en">' + f"{self.name}" + f"" + ) + ] + + if self.description is not None: + outfile["Description"] = [ + ( + f"{add_indent(indent)}" + f'<{ABBR_COM}:Description xml:lang="en">' + f"{self.description}" + f"" + ) + ] + + return outfile + class VersionableArtefact(NameableArtefact, frozen=True, omit_defaults=True): """Versionable Artefact class. @@ -133,6 +222,22 @@ class VersionableArtefact(NameableArtefact, frozen=True, omit_defaults=True): valid_from: Optional[datetime] = None valid_to: Optional[datetime] = None + def _to_XML(self, indent: str) -> Dict[str, List[str]]: + outfile = super(VersionableArtefact, self)._to_XML(indent) + + if self.version is not None: + outfile["Attributes"] += f" version={self.version!r}" + + if self.valid_from is not None: + valid_from_str = self.valid_from.strftime("%Y-%m-%dT%H:%M:%S") + outfile["Attributes"] += f" validFrom={valid_from_str!r}" + + if self.valid_to is not None: + valid_to_str = self.valid_to.strftime("%Y-%m-%dT%H:%M:%S") + outfile["Attributes"] += f" validTo={valid_to_str!r}" + + return outfile + class Item(NameableArtefact, frozen=True, omit_defaults=True): """Item class. @@ -143,6 +248,30 @@ class Item(NameableArtefact, frozen=True, omit_defaults=True): Parent and child attributes (hierarchy) have been removed for simplicity. """ + def _to_XML(self, indent: str) -> str: # type: ignore[override] + head = type(self).__name__ + + if head == "HierarchicalCode": + head = "Code" + + head = f"{ABBR_STR}:" + head + + data = super(Item, self)._to_XML(indent) + outfile = f'{indent}<{head}{data["Attributes"]}>' + outfile += export_intern_data(data, add_indent(indent)) + outfile += f"{indent}" + # if self.parent is not None: + # indent_par = add_indent(indent) + # indent_ref = add_indent(indent_par) + # outfile += f"{indent_par}<{ABBR_STR}:Parent>" + # if isinstance(self.parent, Item): + # text = self.parent.id + # else: + # text = self.parent + # outfile += f'{indent_ref}' + # outfile += f"{indent_par}" + return outfile + class Contact(Struct, frozen=True, omit_defaults=True): """Contact details such as the name of a contact and his email address. @@ -239,6 +368,26 @@ def __post_init__(self) -> None: "Maintainable artefacts must reference an agency.", ) + def _to_XML(self, indent: str) -> Dict[str, Any]: + outfile = super(MaintainableArtefact, self)._to_XML(indent) + + if self.is_external_reference is not None: + outfile["Attributes"] += ( + f" isExternalReference=" + f"{str(self.is_external_reference).lower()!r}" + ) + + if self.is_final is not None: + outfile["Attributes"] += f" isFinal={str(self.is_final).lower()!r}" + + if self.agency is not None: + if isinstance(self.agency, str): + outfile["Attributes"] += f" agencyID={self.agency!r}" + else: + outfile["Attributes"] += f" agencyID={self.agency.id!r}" + + return outfile + class ItemScheme(MaintainableArtefact, frozen=True, omit_defaults=True): """ItemScheme class. @@ -254,6 +403,37 @@ class ItemScheme(MaintainableArtefact, frozen=True, omit_defaults=True): items: Sequence[Item] = () is_partial: bool = False + def _to_XML(self, indent: str) -> str: # type: ignore[override] + """Convert the item scheme to an XML string.""" + indent = add_indent(indent) + + label = f"{ABBR_STR}:{type(self).__name__}" + + data = super(ItemScheme, self)._to_XML(indent) + + if self.is_partial is not None: + data[ + "Attributes" + ] += f" isPartial={str(self.is_partial).lower()!r}" + + outfile = "" + + attributes = data.get("Attributes") or None + + if attributes is not None: + outfile += f"{indent}<{label}{attributes}>" + else: + outfile += f"{indent}<{label}>" + + outfile += export_intern_data(data, indent) + + for item in self.items: + outfile += item._to_XML(add_indent(indent)) + + outfile += f"{indent}" + + return outfile + class DataflowRef(MaintainableArtefact, frozen=True, omit_defaults=True): """Provide core information about a dataflow. diff --git a/src/pysdmx/writers/__write_aux.py b/src/pysdmx/writers/__write_aux.py index cbcafe7..95a9358 100644 --- a/src/pysdmx/writers/__write_aux.py +++ b/src/pysdmx/writers/__write_aux.py @@ -1,5 +1,6 @@ """Writer auxiliary functions.""" +from collections import OrderedDict from datetime import datetime from typing import Any, Dict @@ -225,22 +226,24 @@ def __write_metadata_element( if key in package: outfile += f"{nl}{child2}<{ABBR_STR}:{MSG_CONTENT_PKG[key]}>" - # TODO: Add the __to_XML method to the Item and ItemScheme classes - # for item in package[key].values(): - # outfile += item.__to_XML(MSG_CONTENT_ITEM[key], prettyprint) + for item in package[key].values(): + outfile += item._to_XML(f"{nl}{child2}") outfile += f"{nl}{child2}" return outfile -MSG_CONTENT_PKG = { - ORGS: "OrganisationSchemes", - DATAFLOWS: "Dataflows", - CODELISTS: "Codelists", - CONCEPTS: "Concepts", - DSDS: "DataStructures", - CONSTRAINTS: "Constraints", -} +MSG_CONTENT_PKG = OrderedDict( + [ + (ORGS, "OrganisationSchemes"), + (DATAFLOWS, "Dataflows"), + (CODELISTS, "Codelists"), + (CONCEPTS, "Concepts"), + (DSDS, "DataStructures"), + (CONSTRAINTS, "ContentConstraints"), + ] +) + MSG_CONTENT_ITEM = { ORGS: "AgencyScheme", @@ -287,3 +290,49 @@ def get_end_message(type_: MessageType, prettyprint: bool) -> str: """ nl = "\n" if prettyprint else "" return f"{nl}" + + +def add_indent(indent: str) -> str: + """Adds another indent. + + Args: + indent: The string to be indented + + Returns: + A string with one more indentation + """ + return indent + "\t" + + +def get_outfile(obj_: Dict[str, Any], key: str = "", indent: str = "") -> str: + """Generates an outfile from the object. + + Args: + obj_: The object to be used + key: The key to be used + indent: The indentation to be used + + Returns: + A string with the outfile + + """ + element = obj_.get(key) or [] + + return "".join(element) + + +def export_intern_data(data: Dict[str, Any], indent: str) -> str: + """Export internal data (Annotations, Name, Description) on the XML file. + + Args: + data: Information to be exported + indent: Indentation used + + Returns: + The XML string with the exported data + """ + outfile = get_outfile(data, "Annotations", indent) + outfile += get_outfile(data, "Name", indent) + outfile += get_outfile(data, "Description", indent) + + return outfile diff --git a/src/pysdmx/writers/write.py b/src/pysdmx/writers/write.py index b0cc80e..0a31b9f 100644 --- a/src/pysdmx/writers/write.py +++ b/src/pysdmx/writers/write.py @@ -2,6 +2,7 @@ from typing import Any, Dict, Optional +from pysdmx.model import Code, Codelist, Concept, ConceptScheme from pysdmx.model.message import MessageType from pysdmx.writers.__write_aux import ( create_namespaces, From cc3e06b3e0e0214c634d83a1dc34c2a6e89b1a05 Mon Sep 17 00:00:00 2001 From: "javier.hernandez" Date: Wed, 15 May 2024 13:13:45 +0200 Subject: [PATCH 3/9] Added tests for XML writers. Commented unused data writing methods. Added basic Header class. Signed-off-by: javier.hernandez --- src/pysdmx/model/__base.py | 87 ++++++------- src/pysdmx/writers/__write_aux.py | 93 +++++++------- src/pysdmx/writers/write.py | 104 +++++++++++++++- tests/writers/__init__.py | 0 tests/writers/samples/codelist.xml | 43 +++++++ tests/writers/samples/concept.xml | 30 +++++ tests/writers/samples/empty.xml | 13 ++ tests/writers/test_metadata_writing.py | 162 +++++++++++++++++++++++++ 8 files changed, 432 insertions(+), 100 deletions(-) create mode 100644 tests/writers/__init__.py create mode 100644 tests/writers/samples/codelist.xml create mode 100644 tests/writers/samples/concept.xml create mode 100644 tests/writers/samples/empty.xml create mode 100644 tests/writers/test_metadata_writing.py diff --git a/src/pysdmx/model/__base.py b/src/pysdmx/model/__base.py index 2ad450a..bb29f6b 100644 --- a/src/pysdmx/model/__base.py +++ b/src/pysdmx/model/__base.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from datetime import datetime from typing import Any, Dict, List, Optional, Sequence, Union @@ -63,12 +64,14 @@ def __str__(self) -> str: return ", ".join(out) -ANNOTATION_WRITER = { - "title": "AnnotationTitle", - "type": "AnnotationType", - "url": "AnnotationURL", - "text": "AnnotationText", -} +ANNOTATION_WRITER = OrderedDict( + { + "title": "AnnotationTitle", + "type": "AnnotationType", + "text": "AnnotationText", + "url": "AnnotationURL", + } +) class AnnotableArtefact(Struct, frozen=True, omit_defaults=True, kw_only=True): @@ -107,28 +110,33 @@ def _to_XML(self, indent: str) -> Any: child1 = indent child2 = add_indent(child1) + child3 = add_indent(child2) - outfile = f"<{ABBR_COM}:Annotations>" + outfile = f"{child1}<{ABBR_COM}:Annotations>" for annotation in self.annotations: if annotation.id is None: - outfile += f"{child1}<{ABBR_COM}:Annotation>" + outfile += f"{child2}<{ABBR_COM}:Annotation>" else: outfile += ( - f"{child1}<{ABBR_COM}:Annotation " f"id={annotation.id!r}>" + f"{child2}<{ABBR_COM}:Annotation " f"id={annotation.id!r}>" ) for attr, label in ANNOTATION_WRITER.items(): if getattr(annotation, attr, None) is not None: value = getattr(annotation, attr) value = value.replace("&", "&").rstrip() + if attr == "text": + head_tag = f'{ABBR_COM}:{label} xml:lang="en"' + else: + head_tag = f"{ABBR_COM}:{label}" outfile += ( - f"{child2}<{ABBR_COM}:{label}>" + f"{child3}<{head_tag}>" f"{value}" f"" ) - outfile += f"{child1}" - outfile += f"" + outfile += f"{child2}" + outfile += f"{child1}" return outfile @@ -151,8 +159,7 @@ class IdentifiableArtefact(AnnotableArtefact, frozen=True, omit_defaults=True): def _to_XML(self, indent: str) -> Dict[str, Any]: attributes = "" - if self.id is not None: - attributes += f" id={self.id!r}" + attributes += f" id={self.id!r}" if self.uri is not None: attributes += f" uri={self.uri!r}" @@ -187,7 +194,7 @@ def _to_XML(self, indent: str) -> Dict[str, Any]: if self.name is not None: outfile["Name"] = [ ( - f"{add_indent(indent)}" + f"{indent}" f'<{ABBR_COM}:Name xml:lang="en">' f"{self.name}" f"" @@ -197,7 +204,7 @@ def _to_XML(self, indent: str) -> Dict[str, Any]: if self.description is not None: outfile["Description"] = [ ( - f"{add_indent(indent)}" + f"{indent}" f'<{ABBR_COM}:Description xml:lang="en">' f"{self.description}" f"" @@ -223,10 +230,9 @@ class VersionableArtefact(NameableArtefact, frozen=True, omit_defaults=True): valid_to: Optional[datetime] = None def _to_XML(self, indent: str) -> Dict[str, List[str]]: - outfile = super(VersionableArtefact, self)._to_XML(indent) + outfile = super(VersionableArtefact, self)._to_XML(add_indent(indent)) - if self.version is not None: - outfile["Attributes"] += f" version={self.version!r}" + outfile["Attributes"] += f" version={self.version!r}" if self.valid_from is not None: valid_from_str = self.valid_from.strftime("%Y-%m-%dT%H:%M:%S") @@ -249,14 +255,9 @@ class Item(NameableArtefact, frozen=True, omit_defaults=True): """ def _to_XML(self, indent: str) -> str: # type: ignore[override] - head = type(self).__name__ - - if head == "HierarchicalCode": - head = "Code" + head = f"{ABBR_STR}:" + type(self).__name__ - head = f"{ABBR_STR}:" + head - - data = super(Item, self)._to_XML(indent) + data = super(Item, self)._to_XML(add_indent(indent)) outfile = f'{indent}<{head}{data["Attributes"]}>' outfile += export_intern_data(data, add_indent(indent)) outfile += f"{indent}" @@ -371,20 +372,17 @@ def __post_init__(self) -> None: def _to_XML(self, indent: str) -> Dict[str, Any]: outfile = super(MaintainableArtefact, self)._to_XML(indent) - if self.is_external_reference is not None: - outfile["Attributes"] += ( - f" isExternalReference=" - f"{str(self.is_external_reference).lower()!r}" - ) + outfile["Attributes"] += ( + f" isExternalReference=" + f"{str(self.is_external_reference).lower()!r}" + ) - if self.is_final is not None: - outfile["Attributes"] += f" isFinal={str(self.is_final).lower()!r}" + outfile["Attributes"] += f" isFinal={str(self.is_final).lower()!r}" - if self.agency is not None: - if isinstance(self.agency, str): - outfile["Attributes"] += f" agencyID={self.agency!r}" - else: - outfile["Attributes"] += f" agencyID={self.agency.id!r}" + if isinstance(self.agency, str): + outfile["Attributes"] += f" agencyID={self.agency!r}" + else: + outfile["Attributes"] += f" agencyID={self.agency.id!r}" return outfile @@ -411,19 +409,14 @@ def _to_XML(self, indent: str) -> str: # type: ignore[override] data = super(ItemScheme, self)._to_XML(indent) - if self.is_partial is not None: - data[ - "Attributes" - ] += f" isPartial={str(self.is_partial).lower()!r}" + data["Attributes"] += f" isPartial={str(self.is_partial).lower()!r}" outfile = "" - attributes = data.get("Attributes") or None + attributes = data.get("Attributes") or "" + attributes = attributes.replace("'", '"') - if attributes is not None: - outfile += f"{indent}<{label}{attributes}>" - else: - outfile += f"{indent}<{label}>" + outfile += f"{indent}<{label}{attributes}>" outfile += export_intern_data(data, indent) diff --git a/src/pysdmx/writers/__write_aux.py b/src/pysdmx/writers/__write_aux.py index 95a9358..c316492 100644 --- a/src/pysdmx/writers/__write_aux.py +++ b/src/pysdmx/writers/__write_aux.py @@ -51,39 +51,40 @@ def __namespaces_from_type(type_: MessageType) -> str: Returns: A string with the namespaces """ - if type_ == MessageType.GenericDataSet: - return f"xmlns:{ABBR_GEN}={NAMESPACES[ABBR_GEN]!r} " - elif type_ == MessageType.StructureSpecificDataSet: - return f"xmlns:{ABBR_SPE}={NAMESPACES[ABBR_SPE]!r} " - elif type_ == MessageType.Metadata: - return f"xmlns:{ABBR_STR}={NAMESPACES[ABBR_STR]!r} " - else: - return "" - - -def __namespaces_from_content(content: Dict[str, Any]) -> str: - """Returns the namespaces for the XML file based on content. - - Args: - content: Datasets or None - - Returns: - A string with the namespaces - - Raises: - Exception: If the dataset has no structure defined - """ - outfile = "" - for i, key in enumerate(content): - if content[key].structure is None: - raise Exception(f"Dataset {key} has no structure defined") - ds_urn = URN_DS_BASE - ds_urn += ( - f"{content[key].structure.unique_id}:" - f"ObsLevelDim:{content[key].dim_at_obs}" - ) - outfile += f"xmlns:ns{i}={ds_urn!r}" - return outfile + # if type_ == MessageType.GenericDataSet: + # return f"xmlns:{ABBR_GEN}={NAMESPACES[ABBR_GEN]!r} " + # elif type_ == MessageType.StructureSpecificDataSet: + # return f"xmlns:{ABBR_SPE}={NAMESPACES[ABBR_SPE]!r} " + # elif type_ == MessageType.Metadata: + # return f"xmlns:{ABBR_STR}={NAMESPACES[ABBR_STR]!r} " + # else: + # return "" + return f"xmlns:{ABBR_STR}={NAMESPACES[ABBR_STR]!r} " + + +# def __namespaces_from_content(content: Dict[str, Any]) -> str: +# """Returns the namespaces for the XML file based on content. +# +# Args: +# content: Datasets or None +# +# Returns: +# A string with the namespaces +# +# Raises: +# Exception: If the dataset has no structure defined +# """ +# outfile = "" +# for i, key in enumerate(content): +# if content[key].structure is None: +# raise Exception(f"Dataset {key} has no structure defined") +# ds_urn = URN_DS_BASE +# ds_urn += ( +# f"{content[key].structure.unique_id}:" +# f"ObsLevelDim:{content[key].dim_at_obs}" +# ) +# outfile += f"xmlns:ns{i}={ds_urn!r}" +# return outfile def create_namespaces( @@ -107,8 +108,6 @@ def create_namespaces( outfile += f'xmlns:xsi={NAMESPACES["xsi"]!r} ' outfile += f"xmlns:{ABBR_MSG}={NAMESPACES[ABBR_MSG]!r} " outfile += __namespaces_from_type(type_) - if type_ == MessageType.StructureSpecificDataSet: - outfile += __namespaces_from_content(content) outfile += ( f"xmlns:{ABBR_COM}={NAMESPACES[ABBR_COM]!r} " f'xsi:schemaLocation="{NAMESPACES[ABBR_MSG]} ' @@ -118,10 +117,12 @@ def create_namespaces( return outfile +# We use this point on time to ensure it is fixed on tests +PREPARED_DEFAULT = datetime.strptime("2021-01-01", "%Y-%m-%d") DEFAULT_HEADER = { "ID": "test", "Test": "true", - "Prepared": datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + "Prepared": PREPARED_DEFAULT.strftime("%Y-%m-%dT%H:%M:%S"), "Sender": "Unknown", "Receiver": "Not_Supplied", "DataSetAction": ActionType.Information.value, @@ -165,10 +166,10 @@ def __generate_item_element(element: str, prettyprint: bool) -> str: ) -def __generate_structure_element( - content: Dict[str, Any], prettyprint: bool -) -> str: - return "" +# def __generate_structure_element( +# content: Dict[str, Any], prettyprint: bool +# ) -> str: +# return "" def generate_new_header( @@ -183,13 +184,7 @@ def generate_new_header( Returns: A string with the header - - Raises: - NotImplementedError: If the MessageType is not Metadata """ - if type_ != MessageType.Metadata: - raise NotImplementedError("Only Metadata messages are supported") - nl = "\n" if prettyprint else "" child1 = "\t" if prettyprint else "" @@ -199,9 +194,9 @@ def generate_new_header( outfile += __generate_value_element("Prepared", prettyprint) outfile += __generate_item_element("Sender", prettyprint) outfile += __generate_item_element("Receiver", prettyprint) - if type_.value < MessageType.Metadata.value: - outfile += __generate_structure_element(datasets, prettyprint) - outfile += __generate_value_element("DataSetAction", prettyprint) + # if type_.value < MessageType.Metadata.value: + # outfile += __generate_structure_element(datasets, prettyprint) + # outfile += __generate_value_element("DataSetAction", prettyprint) outfile += __generate_value_element("Source", prettyprint) outfile += f"{nl}{child1}" return outfile diff --git a/src/pysdmx/writers/write.py b/src/pysdmx/writers/write.py index 0a31b9f..2d6eb3b 100644 --- a/src/pysdmx/writers/write.py +++ b/src/pysdmx/writers/write.py @@ -1,10 +1,14 @@ """Writing SDMX-ML files from Message content.""" +from datetime import datetime from typing import Any, Dict, Optional -from pysdmx.model import Code, Codelist, Concept, ConceptScheme +from msgspec import Struct + +from pysdmx.errors import ClientError from pysdmx.model.message import MessageType from pysdmx.writers.__write_aux import ( + ABBR_MSG, create_namespaces, generate_new_header, generate_structures, @@ -12,11 +16,96 @@ ) +class Header(Struct, frozen=True, kw_only=True): + """Header for the SDMX-ML file.""" + + id: str + test: str = "true" + prepared: datetime = datetime.strptime("2021-01-01", "%Y-%m-%d") + sender: str + receiver: str + source: str + + def __post_init__(self) -> None: + """Additional validation checks for Headers.""" + if self.test not in {"true", "false"}: + raise ClientError( + 422, + "Invalid value for 'Test' in Header", + "The Test value must be either 'true' or 'false'", + ) + + @staticmethod + def __value(element: str, value: str, prettyprint: bool) -> str: + """Generates a value element for the XML file. + + A Value element is an XML tag with a value. + + Args: + element: ID, Test, Prepared, Sender, Receiver, Source + value: The value to be written + prettyprint: Prettyprint or not + + Returns: + A string with the value element + """ + nl = "\n" if prettyprint else "" + child2 = "\t\t" if prettyprint else "" + return ( + f"{nl}{child2}<{ABBR_MSG}:{element}>" + f"{value}" + f"" + ) + + @staticmethod + def __item(element: str, id_: str, prettyprint: bool) -> str: + """Generates an item element for the XML file. + + An Item element is an XML tag with an id attribute. + + Args: + element: Sender, Receiver + id_: The ID to be written + prettyprint: Prettyprint or not + + Returns: + A string with the item element + """ + nl = "\n" if prettyprint else "" + child2 = "\t\t" if prettyprint else "" + return f"{nl}{child2}<{ABBR_MSG}:{element} id={id_!r}/>" + + def to_xml(self, prettyprint: bool = True) -> str: + """Converts the Header to an XML string. + + Args: + prettyprint: Prettyprint or not + + Returns: + The XML string + """ + nl = "\n" if prettyprint else "" + child1 = "\t" if prettyprint else "" + prepared = self.prepared.strftime("%Y-%m-%dT%H:%M:%S") + + return ( + f"{nl}{child1}<{ABBR_MSG}:Header>" + f"{self.__value('ID', self.id, prettyprint)}" + f"{self.__value('Test', self.test, prettyprint)}" + f"{self.__value('Prepared', prepared, prettyprint)}" + f"{self.__item('Sender', self.sender, prettyprint)}" + f"{self.__item('Receiver', self.receiver, prettyprint)}" + f"{self.__value('Source', self.source, prettyprint)}" + f"{nl}{child1}" + ) + + def writer( content: Dict[str, Any], type_: MessageType, path: str = "", prettyprint: bool = True, + header: Optional[Header] = None, ) -> Optional[str]: """This function writes a SDMX-ML file from the Message Content. @@ -25,17 +114,24 @@ def writer( type_: The type of message to be written path: The path to save the file prettyprint: Prettyprint or not + header: The header to be used (generated if None) Returns: The XML string if path is empty, None otherwise + Raises: + NotImplementedError: If the MessageType is not Metadata """ + if type_ != MessageType.Metadata: + raise NotImplementedError("Only Metadata messages are supported") outfile = create_namespaces(type_, content, prettyprint) - outfile += generate_new_header(type_, content, prettyprint) + if header is None: + outfile += generate_new_header(type_, content, prettyprint) + else: + outfile += header.to_xml(prettyprint) - if type_ == MessageType.Metadata: - outfile += generate_structures(content, prettyprint) + outfile += generate_structures(content, prettyprint) outfile += get_end_message(type_, prettyprint) diff --git a/tests/writers/__init__.py b/tests/writers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/writers/samples/codelist.xml b/tests/writers/samples/codelist.xml new file mode 100644 index 0000000..2ee2fa9 --- /dev/null +++ b/tests/writers/samples/codelist.xml @@ -0,0 +1,43 @@ + + + + ID + true + 2021-01-01T00:00:00 + + + PySDMX + + + + + + + Frequency + text + Frequency + + + text + Frequency + + + Frequency + + + Frequency + + Annual + + + Monthly + + + Quarterly + + + + + + + \ No newline at end of file diff --git a/tests/writers/samples/concept.xml b/tests/writers/samples/concept.xml new file mode 100644 index 0000000..d6cd621 --- /dev/null +++ b/tests/writers/samples/concept.xml @@ -0,0 +1,30 @@ + + + + ID + true + 2021-01-01T00:00:00 + + + PySDMX + + + + + Frequency + + Annual + Annual + + + Monthly + Monthly + + + Quarterly + Quarterly + + + + + \ No newline at end of file diff --git a/tests/writers/samples/empty.xml b/tests/writers/samples/empty.xml new file mode 100644 index 0000000..504b8c8 --- /dev/null +++ b/tests/writers/samples/empty.xml @@ -0,0 +1,13 @@ + + + + test + true + 2021-01-01T00:00:00 + + + PySDMX + + + + \ No newline at end of file diff --git a/tests/writers/test_metadata_writing.py b/tests/writers/test_metadata_writing.py new file mode 100644 index 0000000..6444baa --- /dev/null +++ b/tests/writers/test_metadata_writing.py @@ -0,0 +1,162 @@ +from datetime import datetime +from pathlib import Path + +import pytest + +from pysdmx.errors import ClientError +from pysdmx.model import Agency, Code, Codelist, Concept, ConceptScheme +from pysdmx.model.__base import Annotation +from pysdmx.model.message import MessageType +from pysdmx.writers.write import Header, writer + +TEST_CS_URN = ( + "urn:sdmx:org.sdmx.infomodel.conceptscheme." + "ConceptScheme=BIS:CS_FREQ(1.0)" +) + + +@pytest.fixture() +def codelist_sample(): + base_path = Path(__file__).parent / "samples" / "codelist.xml" + with open(base_path, "r") as f: + return f.read() + + +@pytest.fixture() +def concept_sample(): + base_path = Path(__file__).parent / "samples" / "concept.xml" + with open(base_path, "r") as f: + return f.read() + + +@pytest.fixture() +def empty_sample(): + base_path = Path(__file__).parent / "samples" / "empty.xml" + with open(base_path, "r") as f: + return f.read() + + +@pytest.fixture() +def header(): + return Header( + id="ID", + sender="Unknown", + receiver="Not_Supplied", + source="PySDMX", + prepared=datetime.strptime("2021-01-01", "%Y-%m-%d"), + ) + + +def test_codelist(codelist_sample, header): + codelist = Codelist( + annotations=[ + Annotation( + id="FREQ_ANOT", + title="Frequency", + text="Frequency", + type="text", + ), + Annotation( + text="Frequency", + type="text", + ), + Annotation( + id="FREQ_ANOT2", + title="Frequency", + ), + ], + id="CL_FREQ", + name="Frequency", + items=[ + Code(id="A", name="Annual"), + Code(id="M", name="Monthly"), + Code(id="Q", name="Quarterly"), + Code(id="W"), + ], + agency="BIS", + version="1.0", + valid_from=datetime.strptime("2021-01-01", "%Y-%m-%d"), + valid_to=datetime.strptime("2021-12-31", "%Y-%m-%d"), + ) + + result = writer( + {"Codelists": {"CL_FREQ": codelist}}, + MessageType.Metadata, + header=header, + ) + + assert result == codelist_sample + + +def test_concept(concept_sample, header): + concept = ConceptScheme( + id="FREQ", + name="Frequency", + agency=Agency(id="BIS"), + version="1.0", + uri=TEST_CS_URN, + urn=TEST_CS_URN, + is_external_reference=False, + is_partial=False, + is_final=False, + items=[ + Concept( + id="A", + name="Annual", + description="Annual", + ), + Concept( + id="M", + name="Monthly", + description="Monthly", + ), + Concept( + id="Q", + name="Quarterly", + description="Quarterly", + ), + ], + ) + + result = writer( + {"Concepts": {"FREQ": concept}}, + MessageType.Metadata, + header=header, + ) + + assert result == concept_sample + + +def test_header_exception(): + with pytest.raises( + ClientError, match="The Test value must be either 'true' or 'false'" + ): + Header( + id="ID", + sender="Unknown", + receiver="Not_Supplied", + source="PySDMX", + prepared=datetime.strptime("2021-01-01", "%Y-%m-%d"), + test="WRONG TEST VALUE", + ) + + +def test_writer_empty(empty_sample): + result = writer({}, MessageType.Metadata, prettyprint=True) + assert result == empty_sample + + +def test_writing_not_supported(): + with pytest.raises( + NotImplementedError, match="Only Metadata messages are supported" + ): + writer({}, MessageType.Error, prettyprint=True) + + +def test_write_to_file(empty_sample, tmpdir): + file = tmpdir.join("output.txt") + result = writer( + {}, MessageType.Metadata, path=file.strpath, prettyprint=True + ) # or use str(file) + assert file.read() == empty_sample + assert result is None From 8acb5b666b1562fe930f672a2d9e6a5e06b0ff75 Mon Sep 17 00:00:00 2001 From: "javier.hernandez" Date: Wed, 15 May 2024 19:07:38 +0200 Subject: [PATCH 4/9] Refactor code to add io module (Xavier suggestion). Refactored code to delete _to_XML methods and have the base package clean. Moved Header class to message.py. Signed-off-by: javier.hernandez --- src/pysdmx/io/__init__.py | 1 + src/pysdmx/io/xml/__init__.py | 1 + src/pysdmx/io/xml/sdmx_two_one/__init__.py | 1 + .../io/xml/sdmx_two_one/writer/__init__.py | 57 ++++ .../xml/sdmx_two_one/writer}/__write_aux.py | 205 ++++--------- .../sdmx_two_one/writer/metadata_writer.py | 276 ++++++++++++++++++ src/pysdmx/model/__base.py | 175 +---------- src/pysdmx/model/message.py | 26 ++ src/pysdmx/writers/__init__.py | 1 - src/pysdmx/writers/write.py | 144 --------- tests/writers/test_metadata_writing.py | 2 +- 11 files changed, 415 insertions(+), 474 deletions(-) create mode 100644 src/pysdmx/io/__init__.py create mode 100644 src/pysdmx/io/xml/__init__.py create mode 100644 src/pysdmx/io/xml/sdmx_two_one/__init__.py create mode 100644 src/pysdmx/io/xml/sdmx_two_one/writer/__init__.py rename src/pysdmx/{writers => io/xml/sdmx_two_one/writer}/__write_aux.py (54%) create mode 100644 src/pysdmx/io/xml/sdmx_two_one/writer/metadata_writer.py delete mode 100644 src/pysdmx/writers/__init__.py delete mode 100644 src/pysdmx/writers/write.py diff --git a/src/pysdmx/io/__init__.py b/src/pysdmx/io/__init__.py new file mode 100644 index 0000000..02b0933 --- /dev/null +++ b/src/pysdmx/io/__init__.py @@ -0,0 +1 @@ +"""IO module for SDMX data.""" diff --git a/src/pysdmx/io/xml/__init__.py b/src/pysdmx/io/xml/__init__.py new file mode 100644 index 0000000..be550c5 --- /dev/null +++ b/src/pysdmx/io/xml/__init__.py @@ -0,0 +1 @@ +"""XML readers and writers.""" diff --git a/src/pysdmx/io/xml/sdmx_two_one/__init__.py b/src/pysdmx/io/xml/sdmx_two_one/__init__.py new file mode 100644 index 0000000..58ed774 --- /dev/null +++ b/src/pysdmx/io/xml/sdmx_two_one/__init__.py @@ -0,0 +1 @@ +"""SDMX 2.1 XML reader and writer.""" diff --git a/src/pysdmx/io/xml/sdmx_two_one/writer/__init__.py b/src/pysdmx/io/xml/sdmx_two_one/writer/__init__.py new file mode 100644 index 0000000..6266155 --- /dev/null +++ b/src/pysdmx/io/xml/sdmx_two_one/writer/__init__.py @@ -0,0 +1,57 @@ +"""SDMX 2.1 writer package.""" + +from typing import Any, Dict, Optional + +from pysdmx.io.xml.sdmx_two_one.writer.__write_aux import ( + __write_header, + create_namespaces, + get_end_message, +) +from pysdmx.io.xml.sdmx_two_one.writer.metadata_writer import ( + generate_structures, +) +from pysdmx.model.message import Header, MessageType + + +def writer( + content: Dict[str, Any], + type_: MessageType, + path: str = "", + prettyprint: bool = True, + header: Optional[Header] = None, +) -> Optional[str]: + """This function writes a SDMX-ML file from the Message Content. + + Args: + content: The content to be written + type_: The type of message to be written + path: The path to save the file + prettyprint: Prettyprint or not + header: The header to be used (generated if None) + + Returns: + The XML string if path is empty, None otherwise + + Raises: + NotImplementedError: If the MessageType is not Metadata + """ + if type_ != MessageType.Metadata: + raise NotImplementedError("Only Metadata messages are supported") + outfile = create_namespaces(type_, content, prettyprint) + + if header is None: + header = Header() + + outfile += __write_header(header, prettyprint) + + outfile += generate_structures(content, prettyprint) + + outfile += get_end_message(type_, prettyprint) + + if path == "": + return outfile + + with open(path, "w", encoding="UTF-8", errors="replace") as f: + f.write(outfile) + + return None diff --git a/src/pysdmx/writers/__write_aux.py b/src/pysdmx/io/xml/sdmx_two_one/writer/__write_aux.py similarity index 54% rename from src/pysdmx/writers/__write_aux.py rename to src/pysdmx/io/xml/sdmx_two_one/writer/__write_aux.py index c316492..9dc8525 100644 --- a/src/pysdmx/writers/__write_aux.py +++ b/src/pysdmx/io/xml/sdmx_two_one/writer/__write_aux.py @@ -1,10 +1,9 @@ """Writer auxiliary functions.""" from collections import OrderedDict -from datetime import datetime from typing import Any, Dict -from pysdmx.model.message import ActionType, MessageType +from pysdmx.model.message import Header, MessageType MESSAGE_TYPE_MAPPING = { MessageType.GenericDataSet: "GenericData", @@ -117,117 +116,6 @@ def create_namespaces( return outfile -# We use this point on time to ensure it is fixed on tests -PREPARED_DEFAULT = datetime.strptime("2021-01-01", "%Y-%m-%d") -DEFAULT_HEADER = { - "ID": "test", - "Test": "true", - "Prepared": PREPARED_DEFAULT.strftime("%Y-%m-%dT%H:%M:%S"), - "Sender": "Unknown", - "Receiver": "Not_Supplied", - "DataSetAction": ActionType.Information.value, - "Source": "PySDMX", -} - - -def __generate_value_element(element: str, prettyprint: bool) -> str: - """Generates a value element for the XML file (XML tag with value). - - Args: - element: ID, Test, Prepared, Sender, Receiver, DataSetAction, Source - prettyprint: Prettyprint or not - - Returns: - A string with the value element - """ - nl = "\n" if prettyprint else "" - child2 = "\t\t" if prettyprint else "" - return ( - f"{nl}{child2}<{ABBR_MSG}:{element}>" - f"{DEFAULT_HEADER[element]}" - f"" - ) - - -def __generate_item_element(element: str, prettyprint: bool) -> str: - """Generates an item element for the XML file (XML tag with id attribute). - - Args: - element: Sender, Receiver - prettyprint: Prettyprint or not - - Returns: - A string with the item element - """ - nl = "\n" if prettyprint else "" - child2 = "\t\t" if prettyprint else "" - return ( - f"{nl}{child2}<{ABBR_MSG}:{element} id={DEFAULT_HEADER[element]!r}/>" - ) - - -# def __generate_structure_element( -# content: Dict[str, Any], prettyprint: bool -# ) -> str: -# return "" - - -def generate_new_header( - type_: MessageType, datasets: Dict[str, Any], prettyprint: bool -) -> str: - """Writes the header to the XML file. - - Args: - type_: MessageType to be used - datasets: Datasets or None - prettyprint: Prettyprint or not - - Returns: - A string with the header - """ - nl = "\n" if prettyprint else "" - child1 = "\t" if prettyprint else "" - - outfile = f"{nl}{child1}<{ABBR_MSG}:Header>" - outfile += __generate_value_element("ID", prettyprint) - outfile += __generate_value_element("Test", prettyprint) - outfile += __generate_value_element("Prepared", prettyprint) - outfile += __generate_item_element("Sender", prettyprint) - outfile += __generate_item_element("Receiver", prettyprint) - # if type_.value < MessageType.Metadata.value: - # outfile += __generate_structure_element(datasets, prettyprint) - # outfile += __generate_value_element("DataSetAction", prettyprint) - outfile += __generate_value_element("Source", prettyprint) - outfile += f"{nl}{child1}" - return outfile - - -def __write_metadata_element( - package: Dict[str, Any], key: str, prettyprint: object -) -> str: - """Writes the metadata element to the XML file. - - Args: - package: The package to be written - key: The key to be used - prettyprint: Prettyprint or not - - Returns: - A string with the metadata element - """ - outfile = "" - nl = "\n" if prettyprint else "" - child2 = "\t\t" if prettyprint else "" - - if key in package: - outfile += f"{nl}{child2}<{ABBR_STR}:{MSG_CONTENT_PKG[key]}>" - for item in package[key].values(): - outfile += item._to_XML(f"{nl}{child2}") - outfile += f"{nl}{child2}" - - return outfile - - MSG_CONTENT_PKG = OrderedDict( [ (ORGS, "OrganisationSchemes"), @@ -250,29 +138,6 @@ def __write_metadata_element( } -def generate_structures(content: Dict[str, Any], prettyprint: bool) -> str: - """Writes the structures to the XML file. - - Args: - content: The Message Content to be written - prettyprint: Prettyprint or not - - Returns: - A string with the structures - """ - nl = "\n" if prettyprint else "" - child1 = "\t" if prettyprint else "" - - outfile = f"{nl}{child1}<{ABBR_MSG}:Structures>" - - for key in MSG_CONTENT_PKG: - outfile += __write_metadata_element(content, key, prettyprint) - - outfile += f"{nl}{child1}" - - return outfile - - def get_end_message(type_: MessageType, prettyprint: bool) -> str: """Returns the end message for the XML file. @@ -299,35 +164,67 @@ def add_indent(indent: str) -> str: return indent + "\t" -def get_outfile(obj_: Dict[str, Any], key: str = "", indent: str = "") -> str: - """Generates an outfile from the object. +def __value(element: str, value: str, prettyprint: bool) -> str: + """Generates a value element for the XML file. + + A Value element is an XML tag with a value. Args: - obj_: The object to be used - key: The key to be used - indent: The indentation to be used + element: ID, Test, Prepared, Sender, Receiver, Source + value: The value to be written + prettyprint: Prettyprint or not Returns: - A string with the outfile - + A string with the value element """ - element = obj_.get(key) or [] + nl = "\n" if prettyprint else "" + child2 = "\t\t" if prettyprint else "" + return ( + f"{nl}{child2}<{ABBR_MSG}:{element}>" + f"{value}" + f"" + ) + + +def __item(element: str, id_: str, prettyprint: bool) -> str: + """Generates an item element for the XML file. + + An Item element is an XML tag with an id attribute. + + Args: + element: Sender, Receiver + id_: The ID to be written + prettyprint: Prettyprint or not - return "".join(element) + Returns: + A string with the item element + """ + nl = "\n" if prettyprint else "" + child2 = "\t\t" if prettyprint else "" + return f"{nl}{child2}<{ABBR_MSG}:{element} id={id_!r}/>" -def export_intern_data(data: Dict[str, Any], indent: str) -> str: - """Export internal data (Annotations, Name, Description) on the XML file. +def __write_header(header: Header, prettyprint: bool) -> str: + """Writes the Header part of the message. Args: - data: Information to be exported - indent: Indentation used + header: The Header to be written + prettyprint: Prettyprint or not Returns: - The XML string with the exported data + The XML string """ - outfile = get_outfile(data, "Annotations", indent) - outfile += get_outfile(data, "Name", indent) - outfile += get_outfile(data, "Description", indent) + nl = "\n" if prettyprint else "" + child1 = "\t" if prettyprint else "" + prepared = header.prepared.strftime("%Y-%m-%dT%H:%M:%S") - return outfile + return ( + f"{nl}{child1}<{ABBR_MSG}:Header>" + f"{__value('ID', header.id, prettyprint)}" + f"{__value('Test', header.test, prettyprint)}" + f"{__value('Prepared', prepared, prettyprint)}" + f"{__item('Sender', header.sender, prettyprint)}" + f"{__item('Receiver', header.receiver, prettyprint)}" + f"{__value('Source', header.source, prettyprint)}" + f"{nl}{child1}" + ) diff --git a/src/pysdmx/io/xml/sdmx_two_one/writer/metadata_writer.py b/src/pysdmx/io/xml/sdmx_two_one/writer/metadata_writer.py new file mode 100644 index 0000000..7a69d3d --- /dev/null +++ b/src/pysdmx/io/xml/sdmx_two_one/writer/metadata_writer.py @@ -0,0 +1,276 @@ +"""Module for writing metadata to XML files.""" + +from collections import OrderedDict +from typing import Any, Dict + +from pysdmx.io.xml.sdmx_two_one.writer.__write_aux import ( + ABBR_COM, + ABBR_MSG, + ABBR_STR, + add_indent, + MSG_CONTENT_PKG, +) +from pysdmx.model.__base import ( + AnnotableArtefact, + IdentifiableArtefact, + Item, + ItemScheme, + MaintainableArtefact, + NameableArtefact, + VersionableArtefact, +) + +ANNOTATION_WRITER = OrderedDict( + { + "title": "AnnotationTitle", + "type": "AnnotationType", + "text": "AnnotationText", + "url": "AnnotationURL", + } +) + + +def __write_annotable(annotable: AnnotableArtefact, indent: str) -> str: + + if len(annotable.annotations) == 0: + return "" + + child1 = indent + child2 = add_indent(child1) + child3 = add_indent(child2) + + outfile = f"{child1}<{ABBR_COM}:Annotations>" + for annotation in annotable.annotations: + if annotation.id is None: + outfile += f"{child2}<{ABBR_COM}:Annotation>" + else: + outfile += ( + f"{child2}<{ABBR_COM}:Annotation " f"id={annotation.id!r}>" + ) + + for attr, label in ANNOTATION_WRITER.items(): + if getattr(annotation, attr, None) is not None: + value = getattr(annotation, attr) + value = value.replace("&", "&").rstrip() + if attr == "text": + head_tag = f'{ABBR_COM}:{label} xml:lang="en"' + else: + head_tag = f"{ABBR_COM}:{label}" + outfile += ( + f"{child3}<{head_tag}>" f"{value}" f"" + ) + + outfile += f"{child2}" + outfile += f"{child1}" + return outfile + + +def __write_identifiable( + identifiable: IdentifiableArtefact, indent: str +) -> Dict[str, Any]: + attributes = "" + + attributes += f" id={identifiable.id!r}" + + if identifiable.uri is not None: + attributes += f" uri={identifiable.uri!r}" + + if identifiable.urn is not None: + attributes += f" urn={identifiable.urn!r}" + + outfile = { + "Annotations": __write_annotable(identifiable, indent), + "Attributes": attributes, + } + + return outfile + + +def __write_nameable( + nameable: NameableArtefact, indent: str +) -> Dict[str, Any]: + outfile = __write_identifiable(nameable, indent) + attrs = ["Name", "Description"] + + for attr in attrs: + if getattr(nameable, attr.lower(), None) is not None: + outfile[attr] = [ + ( + f"{indent}" + f'<{ABBR_COM}:{attr} xml:lang="en">' + f"{getattr(nameable, attr.lower())}" + f"" + ) + ] + + return outfile + + +def __write_versionable( + versionable: VersionableArtefact, indent: str +) -> Dict[str, Any]: + outfile = __write_nameable(versionable, add_indent(indent)) + + outfile["Attributes"] += f" version={versionable.version!r}" + + if versionable.valid_from is not None: + valid_from_str = versionable.valid_from.strftime("%Y-%m-%dT%H:%M:%S") + outfile["Attributes"] += f" validFrom={valid_from_str!r}" + + if versionable.valid_to is not None: + valid_to_str = versionable.valid_to.strftime("%Y-%m-%dT%H:%M:%S") + outfile["Attributes"] += f" validTo={valid_to_str!r}" + + return outfile + + +def __write_maintainable( + maintainable: MaintainableArtefact, indent: str +) -> Dict[str, Any]: + outfile = __write_versionable(maintainable, indent) + + outfile["Attributes"] += ( + f" isExternalReference=" + f"{str(maintainable.is_external_reference).lower()!r}" + ) + + outfile["Attributes"] += f" isFinal={str(maintainable.is_final).lower()!r}" + + if isinstance(maintainable.agency, str): + outfile["Attributes"] += f" agencyID={maintainable.agency!r}" + else: + outfile["Attributes"] += f" agencyID={maintainable.agency.id!r}" + + return outfile + + +def __write_item(item: Item, indent: str) -> str: + head = f"{ABBR_STR}:" + type(item).__name__ + + data = __write_nameable(item, add_indent(indent)) + outfile = f'{indent}<{head}{data["Attributes"]}>' + outfile += export_intern_data(data, add_indent(indent)) + outfile += f"{indent}" + # if self.parent is not None: + # indent_par = add_indent(indent) + # indent_ref = add_indent(indent_par) + # outfile += f"{indent_par}<{ABBR_STR}:Parent>" + # if isinstance(self.parent, Item): + # text = self.parent.id + # else: + # text = self.parent + # outfile += f'{indent_ref}' + # outfile += f"{indent_par}" + return outfile + + +def __write_item_scheme(item_scheme: ItemScheme, indent: str) -> str: + + label = f"{ABBR_STR}:{type(item_scheme).__name__}" + + data = __write_maintainable(item_scheme, indent) + + data["Attributes"] += f" isPartial={str(item_scheme.is_partial).lower()!r}" + + outfile = "" + + attributes = data.get("Attributes") or "" + attributes = attributes.replace("'", '"') + + outfile += f"{indent}<{label}{attributes}>" + + outfile += export_intern_data(data, indent) + + for item in item_scheme.items: + outfile += __write_item(item, add_indent(indent)) + + outfile += f"{indent}" + + return outfile + + +def __write_metadata_element( + package: Dict[str, Any], key: str, prettyprint: object +) -> str: + """Writes the metadata element to the XML file. + + Args: + package: The package to be written + key: The key to be used + prettyprint: Prettyprint or not + + Returns: + A string with the metadata element + """ + outfile = "" + nl = "\n" if prettyprint else "" + child2 = "\t\t" if prettyprint else "" + + base_indent = f"{nl}{child2}" + + if key in package: + outfile += f"{base_indent}<{ABBR_STR}:{MSG_CONTENT_PKG[key]}>" + for item_scheme in package[key].values(): + outfile += __write_item_scheme( + item_scheme, add_indent(base_indent) + ) + outfile += f"{base_indent}" + + return outfile + + +def generate_structures(content: Dict[str, Any], prettyprint: bool) -> str: + """Writes the structures to the XML file. + + Args: + content: The Message Content to be written + prettyprint: Prettyprint or not + + Returns: + A string with the structures + """ + nl = "\n" if prettyprint else "" + child1 = "\t" if prettyprint else "" + + outfile = f"{nl}{child1}<{ABBR_MSG}:Structures>" + + for key in MSG_CONTENT_PKG: + outfile += __write_metadata_element(content, key, prettyprint) + + outfile += f"{nl}{child1}" + + return outfile + + +def export_intern_data(data: Dict[str, Any], indent: str) -> str: + """Export internal data (Annotations, Name, Description) on the XML file. + + Args: + data: Information to be exported + indent: Indentation used + + Returns: + The XML string with the exported data + """ + outfile = get_outfile(data, "Annotations", indent) + outfile += get_outfile(data, "Name", indent) + outfile += get_outfile(data, "Description", indent) + + return outfile + + +def get_outfile(obj_: Dict[str, Any], key: str = "", indent: str = "") -> str: + """Generates an outfile from the object. + + Args: + obj_: The object to be used + key: The key to be used + indent: The indentation to be used + + Returns: + A string with the outfile + + """ + element = obj_.get(key) or [] + + return "".join(element) diff --git a/src/pysdmx/model/__base.py b/src/pysdmx/model/__base.py index bb29f6b..69d97e5 100644 --- a/src/pysdmx/model/__base.py +++ b/src/pysdmx/model/__base.py @@ -1,16 +1,9 @@ -from collections import OrderedDict from datetime import datetime -from typing import Any, Dict, List, Optional, Sequence, Union +from typing import Any, Dict, Optional, Sequence, Union from msgspec import Struct from pysdmx.errors import ClientError -from pysdmx.writers.__write_aux import ( - ABBR_COM, - ABBR_STR, - add_indent, - export_intern_data, -) class Annotation(Struct, frozen=True, omit_defaults=True): @@ -64,16 +57,6 @@ def __str__(self) -> str: return ", ".join(out) -ANNOTATION_WRITER = OrderedDict( - { - "title": "AnnotationTitle", - "type": "AnnotationType", - "text": "AnnotationText", - "url": "AnnotationURL", - } -) - - class AnnotableArtefact(Struct, frozen=True, omit_defaults=True, kw_only=True): """Annotable Artefact class. @@ -103,42 +86,6 @@ def __str__(self) -> str: out.append(f"{k}={str(v)}") return ", ".join(out) - def _to_XML(self, indent: str) -> Any: - - if len(self.annotations) == 0: - return "" - - child1 = indent - child2 = add_indent(child1) - child3 = add_indent(child2) - - outfile = f"{child1}<{ABBR_COM}:Annotations>" - for annotation in self.annotations: - if annotation.id is None: - outfile += f"{child2}<{ABBR_COM}:Annotation>" - else: - outfile += ( - f"{child2}<{ABBR_COM}:Annotation " f"id={annotation.id!r}>" - ) - - for attr, label in ANNOTATION_WRITER.items(): - if getattr(annotation, attr, None) is not None: - value = getattr(annotation, attr) - value = value.replace("&", "&").rstrip() - if attr == "text": - head_tag = f'{ABBR_COM}:{label} xml:lang="en"' - else: - head_tag = f"{ABBR_COM}:{label}" - outfile += ( - f"{child3}<{head_tag}>" - f"{value}" - f"" - ) - - outfile += f"{child2}" - outfile += f"{child1}" - return outfile - class IdentifiableArtefact(AnnotableArtefact, frozen=True, omit_defaults=True): """Identifiable Artefact class. @@ -156,24 +103,6 @@ class IdentifiableArtefact(AnnotableArtefact, frozen=True, omit_defaults=True): uri: Optional[str] = None urn: Optional[str] = None - def _to_XML(self, indent: str) -> Dict[str, Any]: - attributes = "" - - attributes += f" id={self.id!r}" - - if self.uri is not None: - attributes += f" uri={self.uri!r}" - - if self.urn is not None: - attributes += f" urn={self.urn!r}" - - outfile = { - "Annotations": super(IdentifiableArtefact, self)._to_XML(indent), - "Attributes": attributes, - } - - return outfile - class NameableArtefact(IdentifiableArtefact, frozen=True, omit_defaults=True): """Nameable Artefact class. @@ -188,31 +117,6 @@ class NameableArtefact(IdentifiableArtefact, frozen=True, omit_defaults=True): name: Optional[str] = None description: Optional[str] = None - def _to_XML(self, indent: str) -> Dict[str, Any]: - outfile = super(NameableArtefact, self)._to_XML(indent) - - if self.name is not None: - outfile["Name"] = [ - ( - f"{indent}" - f'<{ABBR_COM}:Name xml:lang="en">' - f"{self.name}" - f"" - ) - ] - - if self.description is not None: - outfile["Description"] = [ - ( - f"{indent}" - f'<{ABBR_COM}:Description xml:lang="en">' - f"{self.description}" - f"" - ) - ] - - return outfile - class VersionableArtefact(NameableArtefact, frozen=True, omit_defaults=True): """Versionable Artefact class. @@ -229,21 +133,6 @@ class VersionableArtefact(NameableArtefact, frozen=True, omit_defaults=True): valid_from: Optional[datetime] = None valid_to: Optional[datetime] = None - def _to_XML(self, indent: str) -> Dict[str, List[str]]: - outfile = super(VersionableArtefact, self)._to_XML(add_indent(indent)) - - outfile["Attributes"] += f" version={self.version!r}" - - if self.valid_from is not None: - valid_from_str = self.valid_from.strftime("%Y-%m-%dT%H:%M:%S") - outfile["Attributes"] += f" validFrom={valid_from_str!r}" - - if self.valid_to is not None: - valid_to_str = self.valid_to.strftime("%Y-%m-%dT%H:%M:%S") - outfile["Attributes"] += f" validTo={valid_to_str!r}" - - return outfile - class Item(NameableArtefact, frozen=True, omit_defaults=True): """Item class. @@ -254,25 +143,6 @@ class Item(NameableArtefact, frozen=True, omit_defaults=True): Parent and child attributes (hierarchy) have been removed for simplicity. """ - def _to_XML(self, indent: str) -> str: # type: ignore[override] - head = f"{ABBR_STR}:" + type(self).__name__ - - data = super(Item, self)._to_XML(add_indent(indent)) - outfile = f'{indent}<{head}{data["Attributes"]}>' - outfile += export_intern_data(data, add_indent(indent)) - outfile += f"{indent}" - # if self.parent is not None: - # indent_par = add_indent(indent) - # indent_ref = add_indent(indent_par) - # outfile += f"{indent_par}<{ABBR_STR}:Parent>" - # if isinstance(self.parent, Item): - # text = self.parent.id - # else: - # text = self.parent - # outfile += f'{indent_ref}' - # outfile += f"{indent_par}" - return outfile - class Contact(Struct, frozen=True, omit_defaults=True): """Contact details such as the name of a contact and his email address. @@ -369,23 +239,6 @@ def __post_init__(self) -> None: "Maintainable artefacts must reference an agency.", ) - def _to_XML(self, indent: str) -> Dict[str, Any]: - outfile = super(MaintainableArtefact, self)._to_XML(indent) - - outfile["Attributes"] += ( - f" isExternalReference=" - f"{str(self.is_external_reference).lower()!r}" - ) - - outfile["Attributes"] += f" isFinal={str(self.is_final).lower()!r}" - - if isinstance(self.agency, str): - outfile["Attributes"] += f" agencyID={self.agency!r}" - else: - outfile["Attributes"] += f" agencyID={self.agency.id!r}" - - return outfile - class ItemScheme(MaintainableArtefact, frozen=True, omit_defaults=True): """ItemScheme class. @@ -401,32 +254,6 @@ class ItemScheme(MaintainableArtefact, frozen=True, omit_defaults=True): items: Sequence[Item] = () is_partial: bool = False - def _to_XML(self, indent: str) -> str: # type: ignore[override] - """Convert the item scheme to an XML string.""" - indent = add_indent(indent) - - label = f"{ABBR_STR}:{type(self).__name__}" - - data = super(ItemScheme, self)._to_XML(indent) - - data["Attributes"] += f" isPartial={str(self.is_partial).lower()!r}" - - outfile = "" - - attributes = data.get("Attributes") or "" - attributes = attributes.replace("'", '"') - - outfile += f"{indent}<{label}{attributes}>" - - outfile += export_intern_data(data, indent) - - for item in self.items: - outfile += item._to_XML(add_indent(indent)) - - outfile += f"{indent}" - - return outfile - class DataflowRef(MaintainableArtefact, frozen=True, omit_defaults=True): """Provide core information about a dataflow. diff --git a/src/pysdmx/model/message.py b/src/pysdmx/model/message.py index e6bbf10..3a3aba1 100644 --- a/src/pysdmx/model/message.py +++ b/src/pysdmx/model/message.py @@ -4,8 +4,13 @@ can be written. """ +from datetime import datetime from enum import Enum +from msgspec import Struct + +from pysdmx.errors import ClientError + class MessageType(Enum): """MessageType enumeration. @@ -30,3 +35,24 @@ class ActionType(Enum): Replace = "replace" Delete = "delete" Information = "information" + + +class Header(Struct, frozen=True, kw_only=True): + """Header for the SDMX-ML file.""" + + id: str = "test" + test: str = "true" + prepared: datetime = datetime.strptime("2021-01-01", "%Y-%m-%d") + sender: str = "Unknown" + receiver: str = "Not_Supplied" + source: str = "PySDMX" + dataset_action: str = ActionType.Information.value + + def __post_init__(self) -> None: + """Additional validation checks for Headers.""" + if self.test not in {"true", "false"}: + raise ClientError( + 422, + "Invalid value for 'Test' in Header", + "The Test value must be either 'true' or 'false'", + ) diff --git a/src/pysdmx/writers/__init__.py b/src/pysdmx/writers/__init__.py deleted file mode 100644 index fec157a..0000000 --- a/src/pysdmx/writers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Writer module.""" diff --git a/src/pysdmx/writers/write.py b/src/pysdmx/writers/write.py deleted file mode 100644 index 2d6eb3b..0000000 --- a/src/pysdmx/writers/write.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Writing SDMX-ML files from Message content.""" - -from datetime import datetime -from typing import Any, Dict, Optional - -from msgspec import Struct - -from pysdmx.errors import ClientError -from pysdmx.model.message import MessageType -from pysdmx.writers.__write_aux import ( - ABBR_MSG, - create_namespaces, - generate_new_header, - generate_structures, - get_end_message, -) - - -class Header(Struct, frozen=True, kw_only=True): - """Header for the SDMX-ML file.""" - - id: str - test: str = "true" - prepared: datetime = datetime.strptime("2021-01-01", "%Y-%m-%d") - sender: str - receiver: str - source: str - - def __post_init__(self) -> None: - """Additional validation checks for Headers.""" - if self.test not in {"true", "false"}: - raise ClientError( - 422, - "Invalid value for 'Test' in Header", - "The Test value must be either 'true' or 'false'", - ) - - @staticmethod - def __value(element: str, value: str, prettyprint: bool) -> str: - """Generates a value element for the XML file. - - A Value element is an XML tag with a value. - - Args: - element: ID, Test, Prepared, Sender, Receiver, Source - value: The value to be written - prettyprint: Prettyprint or not - - Returns: - A string with the value element - """ - nl = "\n" if prettyprint else "" - child2 = "\t\t" if prettyprint else "" - return ( - f"{nl}{child2}<{ABBR_MSG}:{element}>" - f"{value}" - f"" - ) - - @staticmethod - def __item(element: str, id_: str, prettyprint: bool) -> str: - """Generates an item element for the XML file. - - An Item element is an XML tag with an id attribute. - - Args: - element: Sender, Receiver - id_: The ID to be written - prettyprint: Prettyprint or not - - Returns: - A string with the item element - """ - nl = "\n" if prettyprint else "" - child2 = "\t\t" if prettyprint else "" - return f"{nl}{child2}<{ABBR_MSG}:{element} id={id_!r}/>" - - def to_xml(self, prettyprint: bool = True) -> str: - """Converts the Header to an XML string. - - Args: - prettyprint: Prettyprint or not - - Returns: - The XML string - """ - nl = "\n" if prettyprint else "" - child1 = "\t" if prettyprint else "" - prepared = self.prepared.strftime("%Y-%m-%dT%H:%M:%S") - - return ( - f"{nl}{child1}<{ABBR_MSG}:Header>" - f"{self.__value('ID', self.id, prettyprint)}" - f"{self.__value('Test', self.test, prettyprint)}" - f"{self.__value('Prepared', prepared, prettyprint)}" - f"{self.__item('Sender', self.sender, prettyprint)}" - f"{self.__item('Receiver', self.receiver, prettyprint)}" - f"{self.__value('Source', self.source, prettyprint)}" - f"{nl}{child1}" - ) - - -def writer( - content: Dict[str, Any], - type_: MessageType, - path: str = "", - prettyprint: bool = True, - header: Optional[Header] = None, -) -> Optional[str]: - """This function writes a SDMX-ML file from the Message Content. - - Args: - content: The content to be written - type_: The type of message to be written - path: The path to save the file - prettyprint: Prettyprint or not - header: The header to be used (generated if None) - - Returns: - The XML string if path is empty, None otherwise - - Raises: - NotImplementedError: If the MessageType is not Metadata - """ - if type_ != MessageType.Metadata: - raise NotImplementedError("Only Metadata messages are supported") - outfile = create_namespaces(type_, content, prettyprint) - - if header is None: - outfile += generate_new_header(type_, content, prettyprint) - else: - outfile += header.to_xml(prettyprint) - - outfile += generate_structures(content, prettyprint) - - outfile += get_end_message(type_, prettyprint) - - if path == "": - return outfile - - with open(path, "w", encoding="UTF-8", errors="replace") as f: - f.write(outfile) - - return None diff --git a/tests/writers/test_metadata_writing.py b/tests/writers/test_metadata_writing.py index 6444baa..86dab0d 100644 --- a/tests/writers/test_metadata_writing.py +++ b/tests/writers/test_metadata_writing.py @@ -4,10 +4,10 @@ import pytest from pysdmx.errors import ClientError +from pysdmx.io.xml.sdmx_two_one.writer import Header, writer from pysdmx.model import Agency, Code, Codelist, Concept, ConceptScheme from pysdmx.model.__base import Annotation from pysdmx.model.message import MessageType -from pysdmx.writers.write import Header, writer TEST_CS_URN = ( "urn:sdmx:org.sdmx.infomodel.conceptscheme." From 71869a633a86ebaf0184d41e56e6e20cd395e21b Mon Sep 17 00:00:00 2001 From: "javier.hernandez" Date: Thu, 16 May 2024 11:47:50 +0200 Subject: [PATCH 5/9] Added double quotes in all xml files (better compatibility and readability). Changed file structure. Changed test value on Header to boolean. Changed MessageType.Metadata to MessageType.Structure. Signed-off-by: javier.hernandez --- .../xml/{sdmx_two_one => sdmx21}/__init__.py | 0 .../writer/__init__.py | 6 +- .../writer/__write_aux.py | 43 ++--------- .../writer/structure.py} | 75 ++++++++----------- src/pysdmx/model/message.py | 19 +---- tests/writers/samples/codelist.xml | 18 ++--- tests/writers/samples/concept.xml | 12 +-- tests/writers/samples/empty.xml | 6 +- tests/writers/test_metadata_writing.py | 26 ++----- 9 files changed, 68 insertions(+), 137 deletions(-) rename src/pysdmx/io/xml/{sdmx_two_one => sdmx21}/__init__.py (100%) rename src/pysdmx/io/xml/{sdmx_two_one => sdmx21}/writer/__init__.py (89%) rename src/pysdmx/io/xml/{sdmx_two_one => sdmx21}/writer/__write_aux.py (80%) rename src/pysdmx/io/xml/{sdmx_two_one/writer/metadata_writer.py => sdmx21/writer/structure.py} (87%) diff --git a/src/pysdmx/io/xml/sdmx_two_one/__init__.py b/src/pysdmx/io/xml/sdmx21/__init__.py similarity index 100% rename from src/pysdmx/io/xml/sdmx_two_one/__init__.py rename to src/pysdmx/io/xml/sdmx21/__init__.py diff --git a/src/pysdmx/io/xml/sdmx_two_one/writer/__init__.py b/src/pysdmx/io/xml/sdmx21/writer/__init__.py similarity index 89% rename from src/pysdmx/io/xml/sdmx_two_one/writer/__init__.py rename to src/pysdmx/io/xml/sdmx21/writer/__init__.py index 6266155..48ac273 100644 --- a/src/pysdmx/io/xml/sdmx_two_one/writer/__init__.py +++ b/src/pysdmx/io/xml/sdmx21/writer/__init__.py @@ -2,12 +2,12 @@ from typing import Any, Dict, Optional -from pysdmx.io.xml.sdmx_two_one.writer.__write_aux import ( +from pysdmx.io.xml.sdmx21.writer.__write_aux import ( __write_header, create_namespaces, get_end_message, ) -from pysdmx.io.xml.sdmx_two_one.writer.metadata_writer import ( +from pysdmx.io.xml.sdmx21.writer.structure import ( generate_structures, ) from pysdmx.model.message import Header, MessageType @@ -35,7 +35,7 @@ def writer( Raises: NotImplementedError: If the MessageType is not Metadata """ - if type_ != MessageType.Metadata: + if type_ != MessageType.Structure: raise NotImplementedError("Only Metadata messages are supported") outfile = create_namespaces(type_, content, prettyprint) diff --git a/src/pysdmx/io/xml/sdmx_two_one/writer/__write_aux.py b/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py similarity index 80% rename from src/pysdmx/io/xml/sdmx_two_one/writer/__write_aux.py rename to src/pysdmx/io/xml/sdmx21/writer/__write_aux.py index 9dc8525..9ab4e80 100644 --- a/src/pysdmx/io/xml/sdmx_two_one/writer/__write_aux.py +++ b/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py @@ -8,7 +8,7 @@ MESSAGE_TYPE_MAPPING = { MessageType.GenericDataSet: "GenericData", MessageType.StructureSpecificDataSet: "StructureSpecificData", - MessageType.Metadata: "Structure", + MessageType.Structure: "Structure", } ABBR_MSG = "mes" @@ -50,42 +50,9 @@ def __namespaces_from_type(type_: MessageType) -> str: Returns: A string with the namespaces """ - # if type_ == MessageType.GenericDataSet: - # return f"xmlns:{ABBR_GEN}={NAMESPACES[ABBR_GEN]!r} " - # elif type_ == MessageType.StructureSpecificDataSet: - # return f"xmlns:{ABBR_SPE}={NAMESPACES[ABBR_SPE]!r} " - # elif type_ == MessageType.Metadata: - # return f"xmlns:{ABBR_STR}={NAMESPACES[ABBR_STR]!r} " - # else: - # return "" return f"xmlns:{ABBR_STR}={NAMESPACES[ABBR_STR]!r} " -# def __namespaces_from_content(content: Dict[str, Any]) -> str: -# """Returns the namespaces for the XML file based on content. -# -# Args: -# content: Datasets or None -# -# Returns: -# A string with the namespaces -# -# Raises: -# Exception: If the dataset has no structure defined -# """ -# outfile = "" -# for i, key in enumerate(content): -# if content[key].structure is None: -# raise Exception(f"Dataset {key} has no structure defined") -# ds_urn = URN_DS_BASE -# ds_urn += ( -# f"{content[key].structure.unique_id}:" -# f"ObsLevelDim:{content[key].dim_at_obs}" -# ) -# outfile += f"xmlns:ns{i}={ds_urn!r}" -# return outfile - - def create_namespaces( type_: MessageType, content: Dict[str, Any], prettyprint: bool = False ) -> str: @@ -113,7 +80,7 @@ def create_namespaces( f'https://registry.sdmx.org/schemas/v2_1/SDMXMessage.xsd">' ) - return outfile + return outfile.replace("'", '"') MSG_CONTENT_PKG = OrderedDict( @@ -217,14 +184,14 @@ def __write_header(header: Header, prettyprint: bool) -> str: nl = "\n" if prettyprint else "" child1 = "\t" if prettyprint else "" prepared = header.prepared.strftime("%Y-%m-%dT%H:%M:%S") - + test = str(header.test).lower() return ( f"{nl}{child1}<{ABBR_MSG}:Header>" f"{__value('ID', header.id, prettyprint)}" - f"{__value('Test', header.test, prettyprint)}" + f"{__value('Test', test, prettyprint)}" f"{__value('Prepared', prepared, prettyprint)}" f"{__item('Sender', header.sender, prettyprint)}" f"{__item('Receiver', header.receiver, prettyprint)}" f"{__value('Source', header.source, prettyprint)}" f"{nl}{child1}" - ) + ).replace("'", '"') diff --git a/src/pysdmx/io/xml/sdmx_two_one/writer/metadata_writer.py b/src/pysdmx/io/xml/sdmx21/writer/structure.py similarity index 87% rename from src/pysdmx/io/xml/sdmx_two_one/writer/metadata_writer.py rename to src/pysdmx/io/xml/sdmx21/writer/structure.py index 7a69d3d..a608743 100644 --- a/src/pysdmx/io/xml/sdmx_two_one/writer/metadata_writer.py +++ b/src/pysdmx/io/xml/sdmx21/writer/structure.py @@ -3,7 +3,7 @@ from collections import OrderedDict from typing import Any, Dict -from pysdmx.io.xml.sdmx_two_one.writer.__write_aux import ( +from pysdmx.io.xml.sdmx21.writer.__write_aux import ( ABBR_COM, ABBR_MSG, ABBR_STR, @@ -47,6 +47,7 @@ def __write_annotable(annotable: AnnotableArtefact, indent: str) -> str: outfile += ( f"{child2}<{ABBR_COM}:Annotation " f"id={annotation.id!r}>" ) + outfile = outfile.replace("'", '"') for attr, label in ANNOTATION_WRITER.items(): if getattr(annotation, attr, None) is not None: @@ -148,19 +149,10 @@ def __write_item(item: Item, indent: str) -> str: head = f"{ABBR_STR}:" + type(item).__name__ data = __write_nameable(item, add_indent(indent)) - outfile = f'{indent}<{head}{data["Attributes"]}>' - outfile += export_intern_data(data, add_indent(indent)) + attributes = data["Attributes"].replace("'", '"') + outfile = f"{indent}<{head}{attributes}>" + outfile += __export_intern_data(data, add_indent(indent)) outfile += f"{indent}" - # if self.parent is not None: - # indent_par = add_indent(indent) - # indent_ref = add_indent(indent_par) - # outfile += f"{indent_par}<{ABBR_STR}:Parent>" - # if isinstance(self.parent, Item): - # text = self.parent.id - # else: - # text = self.parent - # outfile += f'{indent_ref}' - # outfile += f"{indent_par}" return outfile @@ -179,7 +171,7 @@ def __write_item_scheme(item_scheme: ItemScheme, indent: str) -> str: outfile += f"{indent}<{label}{attributes}>" - outfile += export_intern_data(data, indent) + outfile += __export_intern_data(data, indent) for item in item_scheme.items: outfile += __write_item(item, add_indent(indent)) @@ -219,30 +211,23 @@ def __write_metadata_element( return outfile -def generate_structures(content: Dict[str, Any], prettyprint: bool) -> str: - """Writes the structures to the XML file. +def __get_outfile(obj_: Dict[str, Any], key: str = "") -> str: + """Generates an outfile from the object. Args: - content: The Message Content to be written - prettyprint: Prettyprint or not + obj_: The object to be used + key: The key to be used Returns: - A string with the structures - """ - nl = "\n" if prettyprint else "" - child1 = "\t" if prettyprint else "" - - outfile = f"{nl}{child1}<{ABBR_MSG}:Structures>" - - for key in MSG_CONTENT_PKG: - outfile += __write_metadata_element(content, key, prettyprint) + A string with the outfile - outfile += f"{nl}{child1}" + """ + element = obj_.get(key) or [] - return outfile + return "".join(element) -def export_intern_data(data: Dict[str, Any], indent: str) -> str: +def __export_intern_data(data: Dict[str, Any], indent: str) -> str: """Export internal data (Annotations, Name, Description) on the XML file. Args: @@ -252,25 +237,31 @@ def export_intern_data(data: Dict[str, Any], indent: str) -> str: Returns: The XML string with the exported data """ - outfile = get_outfile(data, "Annotations", indent) - outfile += get_outfile(data, "Name", indent) - outfile += get_outfile(data, "Description", indent) + outfile = __get_outfile(data, "Annotations") + outfile += __get_outfile(data, "Name") + outfile += __get_outfile(data, "Description") return outfile -def get_outfile(obj_: Dict[str, Any], key: str = "", indent: str = "") -> str: - """Generates an outfile from the object. +def generate_structures(content: Dict[str, Any], prettyprint: bool) -> str: + """Writes the structures to the XML file. Args: - obj_: The object to be used - key: The key to be used - indent: The indentation to be used + content: The Message Content to be written + prettyprint: Prettyprint or not Returns: - A string with the outfile - + A string with the structures """ - element = obj_.get(key) or [] + nl = "\n" if prettyprint else "" + child1 = "\t" if prettyprint else "" - return "".join(element) + outfile = f"{nl}{child1}<{ABBR_MSG}:Structures>" + + for key in MSG_CONTENT_PKG: + outfile += __write_metadata_element(content, key, prettyprint) + + outfile += f"{nl}{child1}" + + return outfile diff --git a/src/pysdmx/model/message.py b/src/pysdmx/model/message.py index 3a3aba1..f4ede4a 100644 --- a/src/pysdmx/model/message.py +++ b/src/pysdmx/model/message.py @@ -9,8 +9,6 @@ from msgspec import Struct -from pysdmx.errors import ClientError - class MessageType(Enum): """MessageType enumeration. @@ -20,7 +18,7 @@ class MessageType(Enum): GenericDataSet = 1 StructureSpecificDataSet = 2 - Metadata = 3 + Structure = 3 Error = 4 Submission = 5 @@ -38,21 +36,12 @@ class ActionType(Enum): class Header(Struct, frozen=True, kw_only=True): - """Header for the SDMX-ML file.""" + """Header for the SDMX messages.""" id: str = "test" - test: str = "true" + test: bool = True prepared: datetime = datetime.strptime("2021-01-01", "%Y-%m-%d") - sender: str = "Unknown" + sender: str = "ZZZ" receiver: str = "Not_Supplied" source: str = "PySDMX" dataset_action: str = ActionType.Information.value - - def __post_init__(self) -> None: - """Additional validation checks for Headers.""" - if self.test not in {"true", "false"}: - raise ClientError( - 422, - "Invalid value for 'Test' in Header", - "The Test value must be either 'true' or 'false'", - ) diff --git a/tests/writers/samples/codelist.xml b/tests/writers/samples/codelist.xml index 2ee2fa9..2a2befa 100644 --- a/tests/writers/samples/codelist.xml +++ b/tests/writers/samples/codelist.xml @@ -1,18 +1,18 @@ - + ID true 2021-01-01T00:00:00 - - + + PySDMX - + Frequency text Frequency @@ -21,21 +21,21 @@ text Frequency - + Frequency Frequency - + Annual - + Monthly - + Quarterly - + diff --git a/tests/writers/samples/concept.xml b/tests/writers/samples/concept.xml index d6cd621..664ca60 100644 --- a/tests/writers/samples/concept.xml +++ b/tests/writers/samples/concept.xml @@ -1,26 +1,26 @@ - + ID true 2021-01-01T00:00:00 - - + + PySDMX Frequency - + Annual Annual - + Monthly Monthly - + Quarterly Quarterly diff --git a/tests/writers/samples/empty.xml b/tests/writers/samples/empty.xml index 504b8c8..25177fa 100644 --- a/tests/writers/samples/empty.xml +++ b/tests/writers/samples/empty.xml @@ -1,11 +1,11 @@ - + test true 2021-01-01T00:00:00 - - + + PySDMX diff --git a/tests/writers/test_metadata_writing.py b/tests/writers/test_metadata_writing.py index 86dab0d..cb7339c 100644 --- a/tests/writers/test_metadata_writing.py +++ b/tests/writers/test_metadata_writing.py @@ -3,8 +3,7 @@ import pytest -from pysdmx.errors import ClientError -from pysdmx.io.xml.sdmx_two_one.writer import Header, writer +from pysdmx.io.xml.sdmx21.writer import Header, writer from pysdmx.model import Agency, Code, Codelist, Concept, ConceptScheme from pysdmx.model.__base import Annotation from pysdmx.model.message import MessageType @@ -40,7 +39,6 @@ def empty_sample(): def header(): return Header( id="ID", - sender="Unknown", receiver="Not_Supplied", source="PySDMX", prepared=datetime.strptime("2021-01-01", "%Y-%m-%d"), @@ -81,7 +79,7 @@ def test_codelist(codelist_sample, header): result = writer( {"Codelists": {"CL_FREQ": codelist}}, - MessageType.Metadata, + MessageType.Structure, header=header, ) @@ -120,29 +118,15 @@ def test_concept(concept_sample, header): result = writer( {"Concepts": {"FREQ": concept}}, - MessageType.Metadata, + MessageType.Structure, header=header, ) assert result == concept_sample -def test_header_exception(): - with pytest.raises( - ClientError, match="The Test value must be either 'true' or 'false'" - ): - Header( - id="ID", - sender="Unknown", - receiver="Not_Supplied", - source="PySDMX", - prepared=datetime.strptime("2021-01-01", "%Y-%m-%d"), - test="WRONG TEST VALUE", - ) - - def test_writer_empty(empty_sample): - result = writer({}, MessageType.Metadata, prettyprint=True) + result = writer({}, MessageType.Structure, prettyprint=True) assert result == empty_sample @@ -156,7 +140,7 @@ def test_writing_not_supported(): def test_write_to_file(empty_sample, tmpdir): file = tmpdir.join("output.txt") result = writer( - {}, MessageType.Metadata, path=file.strpath, prettyprint=True + {}, MessageType.Structure, path=file.strpath, prettyprint=True ) # or use str(file) assert file.read() == empty_sample assert result is None From 33339458ddaeabed9ac371944e5025e2001a622f Mon Sep 17 00:00:00 2001 From: "javier.hernandez" Date: Thu, 16 May 2024 12:53:35 +0200 Subject: [PATCH 6/9] Refactor tests on writers to io module. Set receiver, source and dataset_action as optional on Header. Signed-off-by: javier.hernandez --- src/pysdmx/io/xml/sdmx21/writer/__write_aux.py | 10 +++++++--- src/pysdmx/io/xml/sdmx21/writer/structure.py | 9 +++++++-- src/pysdmx/model/message.py | 7 ++++--- tests/{writers => io}/__init__.py | 0 tests/io/xml/__init__.py | 0 tests/io/xml/sdmx21/__init__.py | 0 tests/io/xml/sdmx21/writer/__init__.py | 0 .../xml/sdmx21/writer}/samples/codelist.xml | 0 .../xml/sdmx21/writer}/samples/concept.xml | 0 .../xml/sdmx21/writer}/samples/empty.xml | 2 -- .../xml/sdmx21/writer/test_structures_writing.py} | 0 11 files changed, 18 insertions(+), 10 deletions(-) rename tests/{writers => io}/__init__.py (100%) create mode 100644 tests/io/xml/__init__.py create mode 100644 tests/io/xml/sdmx21/__init__.py create mode 100644 tests/io/xml/sdmx21/writer/__init__.py rename tests/{writers => io/xml/sdmx21/writer}/samples/codelist.xml (100%) rename tests/{writers => io/xml/sdmx21/writer}/samples/concept.xml (100%) rename tests/{writers => io/xml/sdmx21/writer}/samples/empty.xml (90%) rename tests/{writers/test_metadata_writing.py => io/xml/sdmx21/writer/test_structures_writing.py} (100%) diff --git a/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py b/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py index 9ab4e80..2049ff3 100644 --- a/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py +++ b/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py @@ -1,7 +1,7 @@ """Writer auxiliary functions.""" from collections import OrderedDict -from typing import Any, Dict +from typing import Any, Dict, Optional from pysdmx.model.message import Header, MessageType @@ -131,7 +131,7 @@ def add_indent(indent: str) -> str: return indent + "\t" -def __value(element: str, value: str, prettyprint: bool) -> str: +def __value(element: str, value: Optional[str], prettyprint: bool) -> str: """Generates a value element for the XML file. A Value element is an XML tag with a value. @@ -144,6 +144,8 @@ def __value(element: str, value: str, prettyprint: bool) -> str: Returns: A string with the value element """ + if not value: + return "" nl = "\n" if prettyprint else "" child2 = "\t\t" if prettyprint else "" return ( @@ -153,7 +155,7 @@ def __value(element: str, value: str, prettyprint: bool) -> str: ) -def __item(element: str, id_: str, prettyprint: bool) -> str: +def __item(element: str, id_: Optional[str], prettyprint: bool) -> str: """Generates an item element for the XML file. An Item element is an XML tag with an id attribute. @@ -166,6 +168,8 @@ def __item(element: str, id_: str, prettyprint: bool) -> str: Returns: A string with the item element """ + if not id_: + return "" nl = "\n" if prettyprint else "" child2 = "\t\t" if prettyprint else "" return f"{nl}{child2}<{ABBR_MSG}:{element} id={id_!r}/>" diff --git a/src/pysdmx/io/xml/sdmx21/writer/structure.py b/src/pysdmx/io/xml/sdmx21/writer/structure.py index a608743..118cb5f 100644 --- a/src/pysdmx/io/xml/sdmx21/writer/structure.py +++ b/src/pysdmx/io/xml/sdmx21/writer/structure.py @@ -31,7 +31,7 @@ def __write_annotable(annotable: AnnotableArtefact, indent: str) -> str: - + """Writes the annotations to the XML file.""" if len(annotable.annotations) == 0: return "" @@ -69,6 +69,7 @@ def __write_annotable(annotable: AnnotableArtefact, indent: str) -> str: def __write_identifiable( identifiable: IdentifiableArtefact, indent: str ) -> Dict[str, Any]: + """Writes the IdentifiableArtefact to the XML file.""" attributes = "" attributes += f" id={identifiable.id!r}" @@ -90,6 +91,7 @@ def __write_identifiable( def __write_nameable( nameable: NameableArtefact, indent: str ) -> Dict[str, Any]: + """Writes the NameableArtefact to the XML file.""" outfile = __write_identifiable(nameable, indent) attrs = ["Name", "Description"] @@ -110,6 +112,7 @@ def __write_nameable( def __write_versionable( versionable: VersionableArtefact, indent: str ) -> Dict[str, Any]: + """Writes the VersionableArtefact to the XML file.""" outfile = __write_nameable(versionable, add_indent(indent)) outfile["Attributes"] += f" version={versionable.version!r}" @@ -128,6 +131,7 @@ def __write_versionable( def __write_maintainable( maintainable: MaintainableArtefact, indent: str ) -> Dict[str, Any]: + """Writes the MaintainableArtefact to the XML file.""" outfile = __write_versionable(maintainable, indent) outfile["Attributes"] += ( @@ -146,6 +150,7 @@ def __write_maintainable( def __write_item(item: Item, indent: str) -> str: + """Writes the item to the XML file.""" head = f"{ABBR_STR}:" + type(item).__name__ data = __write_nameable(item, add_indent(indent)) @@ -157,7 +162,7 @@ def __write_item(item: Item, indent: str) -> str: def __write_item_scheme(item_scheme: ItemScheme, indent: str) -> str: - + """Writes the item scheme to the XML file.""" label = f"{ABBR_STR}:{type(item_scheme).__name__}" data = __write_maintainable(item_scheme, indent) diff --git a/src/pysdmx/model/message.py b/src/pysdmx/model/message.py index f4ede4a..1121bce 100644 --- a/src/pysdmx/model/message.py +++ b/src/pysdmx/model/message.py @@ -6,6 +6,7 @@ from datetime import datetime from enum import Enum +from typing import Optional from msgspec import Struct @@ -42,6 +43,6 @@ class Header(Struct, frozen=True, kw_only=True): test: bool = True prepared: datetime = datetime.strptime("2021-01-01", "%Y-%m-%d") sender: str = "ZZZ" - receiver: str = "Not_Supplied" - source: str = "PySDMX" - dataset_action: str = ActionType.Information.value + receiver: Optional[str] = None + source: Optional[str] = None + dataset_action: Optional[ActionType] = None diff --git a/tests/writers/__init__.py b/tests/io/__init__.py similarity index 100% rename from tests/writers/__init__.py rename to tests/io/__init__.py diff --git a/tests/io/xml/__init__.py b/tests/io/xml/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/io/xml/sdmx21/__init__.py b/tests/io/xml/sdmx21/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/io/xml/sdmx21/writer/__init__.py b/tests/io/xml/sdmx21/writer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/writers/samples/codelist.xml b/tests/io/xml/sdmx21/writer/samples/codelist.xml similarity index 100% rename from tests/writers/samples/codelist.xml rename to tests/io/xml/sdmx21/writer/samples/codelist.xml diff --git a/tests/writers/samples/concept.xml b/tests/io/xml/sdmx21/writer/samples/concept.xml similarity index 100% rename from tests/writers/samples/concept.xml rename to tests/io/xml/sdmx21/writer/samples/concept.xml diff --git a/tests/writers/samples/empty.xml b/tests/io/xml/sdmx21/writer/samples/empty.xml similarity index 90% rename from tests/writers/samples/empty.xml rename to tests/io/xml/sdmx21/writer/samples/empty.xml index 25177fa..4454a70 100644 --- a/tests/writers/samples/empty.xml +++ b/tests/io/xml/sdmx21/writer/samples/empty.xml @@ -5,8 +5,6 @@ true 2021-01-01T00:00:00 - - PySDMX diff --git a/tests/writers/test_metadata_writing.py b/tests/io/xml/sdmx21/writer/test_structures_writing.py similarity index 100% rename from tests/writers/test_metadata_writing.py rename to tests/io/xml/sdmx21/writer/test_structures_writing.py From dd2f6cd55c17a03d8eb13c28a22d872bf4149cc1 Mon Sep 17 00:00:00 2001 From: "javier.hernandez" Date: Thu, 16 May 2024 20:04:58 +0200 Subject: [PATCH 7/9] Moved MessageType to io.xml.enums. Signed-off-by: javier.hernandez --- src/pysdmx/io/xml/enums.py | 16 ++++++++++++++++ src/pysdmx/io/xml/sdmx21/writer/__init__.py | 3 ++- src/pysdmx/io/xml/sdmx21/writer/__write_aux.py | 3 ++- src/pysdmx/model/message.py | 13 ------------- .../xml/sdmx21/writer/test_structures_writing.py | 2 +- 5 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 src/pysdmx/io/xml/enums.py diff --git a/src/pysdmx/io/xml/enums.py b/src/pysdmx/io/xml/enums.py new file mode 100644 index 0000000..7803831 --- /dev/null +++ b/src/pysdmx/io/xml/enums.py @@ -0,0 +1,16 @@ +"""Enumeration for the XML message types.""" + +from enum import Enum + + +class MessageType(Enum): + """MessageType enumeration. + + Enumeration that withholds the Message type for writing purposes. + """ + + GenericDataSet = 1 + StructureSpecificDataSet = 2 + Structure = 3 + Error = 4 + Submission = 5 diff --git a/src/pysdmx/io/xml/sdmx21/writer/__init__.py b/src/pysdmx/io/xml/sdmx21/writer/__init__.py index 48ac273..26a2a0c 100644 --- a/src/pysdmx/io/xml/sdmx21/writer/__init__.py +++ b/src/pysdmx/io/xml/sdmx21/writer/__init__.py @@ -2,6 +2,7 @@ from typing import Any, Dict, Optional +from pysdmx.io.xml.enums import MessageType from pysdmx.io.xml.sdmx21.writer.__write_aux import ( __write_header, create_namespaces, @@ -10,7 +11,7 @@ from pysdmx.io.xml.sdmx21.writer.structure import ( generate_structures, ) -from pysdmx.model.message import Header, MessageType +from pysdmx.model.message import Header def writer( diff --git a/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py b/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py index 2049ff3..7b73959 100644 --- a/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py +++ b/src/pysdmx/io/xml/sdmx21/writer/__write_aux.py @@ -3,7 +3,8 @@ from collections import OrderedDict from typing import Any, Dict, Optional -from pysdmx.model.message import Header, MessageType +from pysdmx.io.xml.enums import MessageType +from pysdmx.model.message import Header MESSAGE_TYPE_MAPPING = { MessageType.GenericDataSet: "GenericData", diff --git a/src/pysdmx/model/message.py b/src/pysdmx/model/message.py index 1121bce..529a3a6 100644 --- a/src/pysdmx/model/message.py +++ b/src/pysdmx/model/message.py @@ -11,19 +11,6 @@ from msgspec import Struct -class MessageType(Enum): - """MessageType enumeration. - - Enumeration that withholds the Message type for writing purposes. - """ - - GenericDataSet = 1 - StructureSpecificDataSet = 2 - Structure = 3 - Error = 4 - Submission = 5 - - class ActionType(Enum): """ActionType enumeration. diff --git a/tests/io/xml/sdmx21/writer/test_structures_writing.py b/tests/io/xml/sdmx21/writer/test_structures_writing.py index cb7339c..d2d8d95 100644 --- a/tests/io/xml/sdmx21/writer/test_structures_writing.py +++ b/tests/io/xml/sdmx21/writer/test_structures_writing.py @@ -3,10 +3,10 @@ import pytest +from pysdmx.io.xml.enums import MessageType from pysdmx.io.xml.sdmx21.writer import Header, writer from pysdmx.model import Agency, Code, Codelist, Concept, ConceptScheme from pysdmx.model.__base import Annotation -from pysdmx.model.message import MessageType TEST_CS_URN = ( "urn:sdmx:org.sdmx.infomodel.conceptscheme." From ef582a6136ad0dfce796624e75813c039c91b66e Mon Sep 17 00:00:00 2001 From: "javier.hernandez" Date: Fri, 17 May 2024 13:28:24 +0200 Subject: [PATCH 8/9] Changed default values on Header. Added test for generated Header with defaults. Signed-off-by: javier.hernandez --- src/pysdmx/model/message.py | 5 ++- tests/io/xml/sdmx21/writer/samples/empty.xml | 2 +- .../sdmx21/writer/test_structures_writing.py | 40 ++++++++++++++----- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/pysdmx/model/message.py b/src/pysdmx/model/message.py index 529a3a6..77ba5fe 100644 --- a/src/pysdmx/model/message.py +++ b/src/pysdmx/model/message.py @@ -7,6 +7,7 @@ from datetime import datetime from enum import Enum from typing import Optional +import uuid from msgspec import Struct @@ -26,9 +27,9 @@ class ActionType(Enum): class Header(Struct, frozen=True, kw_only=True): """Header for the SDMX messages.""" - id: str = "test" + id: str = str(uuid.uuid4()) test: bool = True - prepared: datetime = datetime.strptime("2021-01-01", "%Y-%m-%d") + prepared: datetime = datetime.now() sender: str = "ZZZ" receiver: Optional[str] = None source: Optional[str] = None diff --git a/tests/io/xml/sdmx21/writer/samples/empty.xml b/tests/io/xml/sdmx21/writer/samples/empty.xml index 4454a70..21a6016 100644 --- a/tests/io/xml/sdmx21/writer/samples/empty.xml +++ b/tests/io/xml/sdmx21/writer/samples/empty.xml @@ -1,7 +1,7 @@ - test + ID true 2021-01-01T00:00:00 diff --git a/tests/io/xml/sdmx21/writer/test_structures_writing.py b/tests/io/xml/sdmx21/writer/test_structures_writing.py index d2d8d95..03d0913 100644 --- a/tests/io/xml/sdmx21/writer/test_structures_writing.py +++ b/tests/io/xml/sdmx21/writer/test_structures_writing.py @@ -39,13 +39,22 @@ def empty_sample(): def header(): return Header( id="ID", + prepared=datetime.strptime("2021-01-01", "%Y-%m-%d"), + ) + + +@pytest.fixture() +def complete_header(): + return Header( + id="ID", + prepared=datetime.strptime("2021-01-01", "%Y-%m-%d"), + sender="ZZZ", receiver="Not_Supplied", source="PySDMX", - prepared=datetime.strptime("2021-01-01", "%Y-%m-%d"), ) -def test_codelist(codelist_sample, header): +def test_codelist(codelist_sample, complete_header): codelist = Codelist( annotations=[ Annotation( @@ -80,13 +89,13 @@ def test_codelist(codelist_sample, header): result = writer( {"Codelists": {"CL_FREQ": codelist}}, MessageType.Structure, - header=header, + header=complete_header, ) assert result == codelist_sample -def test_concept(concept_sample, header): +def test_concept(concept_sample, complete_header): concept = ConceptScheme( id="FREQ", name="Frequency", @@ -119,14 +128,14 @@ def test_concept(concept_sample, header): result = writer( {"Concepts": {"FREQ": concept}}, MessageType.Structure, - header=header, + header=complete_header, ) assert result == concept_sample -def test_writer_empty(empty_sample): - result = writer({}, MessageType.Structure, prettyprint=True) +def test_writer_empty(empty_sample, header): + result = writer({}, MessageType.Structure, prettyprint=True, header=header) assert result == empty_sample @@ -137,10 +146,23 @@ def test_writing_not_supported(): writer({}, MessageType.Error, prettyprint=True) -def test_write_to_file(empty_sample, tmpdir): +def test_write_to_file(empty_sample, tmpdir, header): file = tmpdir.join("output.txt") result = writer( - {}, MessageType.Structure, path=file.strpath, prettyprint=True + {}, + MessageType.Structure, + path=file.strpath, + prettyprint=True, + header=header, ) # or use str(file) assert file.read() == empty_sample assert result is None + + +def test_writer_no_header(): + result: str = writer({}, MessageType.Structure, prettyprint=False) + assert "" in result + assert "" in result + assert "true" in result + assert "" in result + assert '' in result From 6357096f2b1229f2f1190a926d42dca84f52a25e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20Hern=C3=A1ndez=20del=20Ca=C3=B1o?= Date: Fri, 17 May 2024 12:03:52 +0000 Subject: [PATCH 9/9] Added timezone.utc to default value in Header.prepared --- src/pysdmx/model/message.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pysdmx/model/message.py b/src/pysdmx/model/message.py index 77ba5fe..d9daf33 100644 --- a/src/pysdmx/model/message.py +++ b/src/pysdmx/model/message.py @@ -4,7 +4,7 @@ can be written. """ -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from typing import Optional import uuid @@ -29,7 +29,7 @@ class Header(Struct, frozen=True, kw_only=True): id: str = str(uuid.uuid4()) test: bool = True - prepared: datetime = datetime.now() + prepared: datetime = datetime.now(timezone.utc) sender: str = "ZZZ" receiver: Optional[str] = None source: Optional[str] = None