Merge pull request #3 from bis-med-it/many_attrs

Merge values of attributes with the same ID

src/pysdmx/fmr/fusion/report.py

@@ -5,6 +5,7 @@
 from msgspec import Struct
 
 from pysdmx.fmr.fusion.core import FusionString
+from pysdmx.fmr.reader import _merge_attributes
 from pysdmx.model import MetadataAttribute, MetadataReport
 
 
@@ -32,6 +33,7 @@ class FusionMetadataMessage(Struct, frozen=True):
     def to_model(self) -> MetadataReport:
         """Returns the requested metadata report."""
         r = self.data.metadatasets[0]
+        attrs = _merge_attributes(r.attributes)
         return MetadataReport(
-            r.id, r.names[0].value, r.metadataflow, r.targets, r.attributes
+            r.id, r.names[0].value, r.metadataflow, r.targets, attrs
         )

src/pysdmx/fmr/reader.py

@@ -1,7 +1,10 @@
 """API for FMR readers."""
 
+from collections import defaultdict
 from dataclasses import dataclass
-from typing import Any, Protocol, runtime_checkable
+from typing import Any, Dict, List, Protocol, runtime_checkable, Sequence
+
+from pysdmx.model import MetadataAttribute
 
 
 @runtime_checkable
@@ -27,3 +30,42 @@ class Deserializers:
     report: Deserializer
     mapping: Deserializer
     code_map: Deserializer
+
+
+def _merge_attributes(
+    attrs: Sequence[MetadataAttribute],
+) -> Sequence[MetadataAttribute]:
+    """Groups together the values of attributes with the same ID.
+
+    The function assumes that an attribute will either have a
+    value or will act as a container for other attributes. In
+    case the attribute contains other attributes AND has a value,
+    this function will NOT work as expected.
+
+    Args:
+        attrs: The list of attributes to be merged
+
+    Returns:
+        The list of (possibly merged) attributes
+    """
+    by_id: Dict[str, List[Any]] = defaultdict(list)
+    sub_id = []
+
+    for attr in attrs:
+        if attr.attributes:
+            sub_id.append(
+                MetadataAttribute(
+                    attr.id,
+                    attr.value,
+                    _merge_attributes(attr.attributes),
+                )
+            )
+        else:
+            by_id[attr.id].append(attr.value)
+
+    out = []
+    out.extend(sub_id)
+    for k, v in by_id.items():
+        val = v if len(v) > 1 else v[0]
+        out.append(MetadataAttribute(k, val))
+    return out

src/pysdmx/fmr/sdmx/report.py

@@ -4,6 +4,7 @@
 
 from msgspec import Struct
 
+from pysdmx.fmr.reader import _merge_attributes
 from pysdmx.model import MetadataReport
 
 
@@ -20,4 +21,6 @@ class JsonMetadataMessage(Struct, frozen=True):
 
     def to_model(self) -> MetadataReport:
         """Returns the requested metadata report."""
-        return self.data.metadataSets[0]
+        r = self.data.metadataSets[0]
+        attrs = _merge_attributes(r.attributes)
+        return MetadataReport(r.id, r.name, r.metadataflow, r.targets, attrs)

tests/fmr/fusion/test_report.py

@@ -35,6 +35,21 @@ def body():
         return f.read()
 
 
+@pytest.fixture()
+def query2(fmr):
+    res = "metadata/metadataset/"
+    provider = "BIS.MEDIT"
+    id = "DTI_OCC_SRC"
+    version = "1.0"
+    return f"{fmr.api_endpoint}{res}{provider}/{id}/{version}"
+
+
+@pytest.fixture()
+def body2():
+    with open("tests/fmr/samples/refmeta/report_attrs.fusion.json", "rb") as f:
+        return f.read()
+
+
 def test_returns_report(respx_mock, fmr, query, body):
     """get_hierarchy() should return a metadata report."""
     checks.check_report(respx_mock, fmr, query, body)
@@ -44,3 +59,8 @@ def test_returns_report(respx_mock, fmr, query, body):
 async def test_attributes(respx_mock, async_fmr, query, body):
     """Report contains the expected attributes."""
     await checks.check_attributes(respx_mock, async_fmr, query, body)
+
+
+def test_same_id_attrs(respx_mock, fmr, query2, body2):
+    """Attributes with the same ID are treated as sequence."""
+    checks.check_same_id_attrs(respx_mock, fmr, query2, body2)

tests/fmr/report_checks.py

@@ -59,3 +59,27 @@ async def check_attributes(mock, fmr: AsyncRegistryClient, query, body):
             assert len(attr.attributes) == 0
         else:
             pytest.fail(f"Unexpected attribute: {attr.id}")
+
+
+def check_same_id_attrs(mock, fmr: RegistryClient, query, body):
+    """Attributes with the same ID are treated as sequence."""
+    mock.get(query).mock(
+        return_value=httpx.Response(
+            200,
+            content=body,
+        )
+    )
+
+    report = fmr.get_report("BIS.MEDIT", "DTI_OCC_SRC", "1.0")
+
+    assert len(report) == 2
+    for attr in report:
+        if attr.id == "DF_MANAGED":
+            assert isinstance(attr.value, bool)
+        else:
+            assert len(attr.value) == 2
+            for val in attr.value:
+                assert val in ["CL1", "CL2"]
+    same_ids = report["DF_DYNCL"]
+    assert len(same_ids.value) == 2
+    assert val in ["CL1", "CL2"]

tests/fmr/samples/refmeta/report_attrs.fusion.json

@@ -0,0 +1,52 @@
+{
+    "meta": {
+        "id": "IREF389470",
+        "test": false,
+        "prepared": "2023-09-18T12:09:04Z",
+        "contentLanguages": [
+            "en"
+        ],
+        "sender": {
+            "id": "FusionRegistry"
+        },
+        "format": "fusionjson"
+    },
+    "data": {
+        "metadatasets": [
+            {
+                "id": "DTI_OCC_SRC",
+                "urn": "urn:sdmx:org.sdmx.infomodel.metadatastructure.MetadataSet=BIS.MEDIT:DTI_OCC_SRC(1.0)",
+                "names": [
+                    {
+                        "locale": "en",
+                        "value": "Technical metadata for BIS.XTD:OCC_SRC"
+                    }
+                ],
+                "agencyId": "BIS.MEDIT",
+                "version": "1.0",
+                "isFinal": false,
+                "metadataflow": "urn:sdmx:org.sdmx.infomodel.metadatastructure.Metadataflow=BIS.MEDIT:DTI(1.0)",
+                "targets": [
+                    "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS.XTD:OCC_SRC(1.0)"
+                ],
+                "attributes": [
+                    {
+                        "id": "DF_DYNCL",
+                        "urn": "urn:sdmx:org.sdmx.infomodel.metadatastructure.MetadataAttribute=BIS.MEDIT:DTI_BIS_MACRO(1.0).DF_DYNCL",
+                        "value": "CL1"
+                    },
+                    {
+                        "id": "DF_DYNCL",
+                        "urn": "urn:sdmx:org.sdmx.infomodel.metadatastructure.MetadataAttribute=BIS.MEDIT:DTI_BIS_MACRO(1.0).DF_DYNCL",
+                        "value": "CL2"
+                    },
+                    {
+                        "id": "DF_MANAGED",
+                        "urn": "urn:sdmx:org.sdmx.infomodel.metadatastructure.MetadataAttribute=BIS.MEDIT:DTI_BIS_MACRO(1.0).DF_MANAGED",
+                        "value": true
+                    }
+                ]
+            }
+        ]
+    }
+}

tests/fmr/samples/refmeta/report_attrs.json

@@ -0,0 +1,79 @@
+{
+    "meta": {
+        "id": "IREF944364",
+        "test": false,
+        "schema": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/master/metadata-message/tools/schemas/2.0.0/sdmx-json-metadata-schema.json",
+        "prepared": "2023-08-16T14:00:39Z",
+        "contentLanguages": [
+            "en"
+        ],
+        "sender": {
+            "id": "FusionRegistry"
+        }
+    },
+    "data": {
+        "metadataSets": [
+            {
+                "links": [
+                    {
+                        "rel": "self",
+                        "type": "metadataset",
+                        "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json",
+                        "urn": "urn:sdmx:org.sdmx.infomodel.metadatastructure.MetadataSet=BIS.MEDIT:DTI_OCC_SRC(1.0)"
+                    }
+                ],
+                "id": "DTI_OCC_SRC",
+                "name": "Technical metadata for BIS.XTD:OCC_SRC",
+                "names": {
+                    "en": "Technical metadata for BIS.XTD:OCC_SRC"
+                },
+                "version": "1.0",
+                "agencyID": "BIS.MEDIT",
+                "isExternalReference": false,
+                "isFinal": false,
+                "metadataflow": "urn:sdmx:org.sdmx.infomodel.metadatastructure.Metadataflow=BIS.MEDIT:DTI(1.0)",
+                "targets": [
+                    "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS.XTD:OCC_SRC(1.0)"
+                ],
+                "attributes": [
+                    {
+                        "links": [
+                            {
+                                "rel": "self",
+                                "type": "metadataattribute",
+                                "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json",
+                                "urn": "urn:sdmx:org.sdmx.infomodel.metadatastructure.MetadataAttribute=BIS.MEDIT:DTI_BIS_MACRO(1.0).DF_DYNCL"
+                            }
+                        ],
+                        "id": "DF_DYNCL",
+                        "value": "CL1"
+                    },
+                    {
+                        "links": [
+                            {
+                                "rel": "self",
+                                "type": "metadataattribute",
+                                "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json",
+                                "urn": "urn:sdmx:org.sdmx.infomodel.metadatastructure.MetadataAttribute=BIS.MEDIT:DTI_BIS_MACRO(1.0).DF_DYNCL"
+                            }
+                        ],
+                        "id": "DF_DYNCL",
+                        "value": "CL2"
+                    },
+                    {
+                        "links": [
+                            {
+                                "rel": "self",
+                                "type": "metadataattribute",
+                                "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json",
+                                "urn": "urn:sdmx:org.sdmx.infomodel.metadatastructure.MetadataAttribute=BIS.MEDIT:DTI_BIS_MACRO(1.0).DF_MANAGED"
+                            }
+                        ],
+                        "id": "DF_MANAGED",
+                        "value": true
+                    }
+                ]
+            }
+        ]
+    }
+}

tests/fmr/sdmx/test_report.py

@@ -35,6 +35,21 @@ def body():
         return f.read()
 
 
+@pytest.fixture()
+def query2(fmr):
+    res = "metadata/metadataset/"
+    provider = "BIS.MEDIT"
+    id = "DTI_OCC_SRC"
+    version = "1.0"
+    return f"{fmr.api_endpoint}{res}{provider}/{id}/{version}"
+
+
+@pytest.fixture()
+def body2():
+    with open("tests/fmr/samples/refmeta/report_attrs.json", "rb") as f:
+        return f.read()
+
+
 def test_returns_report(respx_mock, fmr, query, body):
     """get_hierarchy() should return a metadata report."""
     checks.check_report(respx_mock, fmr, query, body)
@@ -44,3 +59,8 @@ def test_returns_report(respx_mock, fmr, query, body):
 async def test_attributes(respx_mock, async_fmr, query, body):
     """Report contains the expected attributes."""
     await checks.check_attributes(respx_mock, async_fmr, query, body)
+
+
+def test_same_id_attrs(respx_mock, fmr, query2, body2):
+    """Attributes with the same ID are treated as sequence."""
+    checks.check_same_id_attrs(respx_mock, fmr, query2, body2)