From 659b389556cc614020f1c792f46a59dccd761f86 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 09:59:59 +0100 Subject: [PATCH 01/26] Component codes can come from a hierarchy --- src/pysdmx/model/dataflow.py | 6 +++--- tests/model/test_component.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pysdmx/model/dataflow.py b/src/pysdmx/model/dataflow.py index 6210167..1236e31 100644 --- a/src/pysdmx/model/dataflow.py +++ b/src/pysdmx/model/dataflow.py @@ -10,11 +10,11 @@ from collections import Counter, UserList from datetime import datetime from enum import Enum -from typing import Any, Iterable, Optional, Sequence +from typing import Any, Iterable, Optional, Sequence, Union from msgspec import Struct -from pysdmx.model.code import Code +from pysdmx.model.code import Code, Codelist, Hierarchy from pysdmx.model.concept import DataType, Facets from pysdmx.model.organisation import Organisation @@ -107,7 +107,7 @@ class Component(Struct, frozen=True, omit_defaults=True): facets: Optional[Facets] = None name: Optional[str] = None description: Optional[str] = None - codes: Sequence[Code] = () + codes: Union[Codelist, Hierarchy, None] = None attachment_level: Optional[str] = None enum_ref: Optional[str] = None array_def: Optional[ArrayBoundaries] = None diff --git a/tests/model/test_component.py b/tests/model/test_component.py index cb7b357..06a6f1e 100644 --- a/tests/model/test_component.py +++ b/tests/model/test_component.py @@ -43,7 +43,7 @@ def test_defaults(fid, req, role): assert f.facets is None assert f.name is None assert f.description is None - assert len(f.codes) == 0 + assert not f.codes assert f.attachment_level is None assert f.enum_ref is None @@ -74,7 +74,7 @@ def test_full_initialization(fid, req, role, typ, cl_ref, array_def): assert f.facets == facets assert f.name == name assert f.description == desc - assert len(f.codes) == 0 + assert not f.codes assert f.attachment_level == lvl assert f.enum_ref == cl_ref assert f.array_def == array_def From 53eb6dc54b59f0cfe4aa32892beb725579310aae Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 10:08:31 +0100 Subject: [PATCH 02/26] enum_ref no longer needed --- src/pysdmx/fmr/fusion/dsd.py | 13 ++++--------- src/pysdmx/fmr/sdmx/dsd.py | 14 ++++---------- src/pysdmx/model/dataflow.py | 3 --- tests/fmr/schema_checks.py | 7 +------ tests/model/test_component.py | 5 +---- 5 files changed, 10 insertions(+), 32 deletions(-) diff --git a/src/pysdmx/fmr/fusion/dsd.py b/src/pysdmx/fmr/fusion/dsd.py index dedf310..5341257 100644 --- a/src/pysdmx/fmr/fusion/dsd.py +++ b/src/pysdmx/fmr/fusion/dsd.py @@ -48,7 +48,6 @@ def _get_representation( ]: valid = cons.get(id_, []) codes: Sequence[Code] = [] - cl_ref = None ab = None dt = DataType.STRING facets = None @@ -64,9 +63,8 @@ def _get_representation( dt = DataType(r.textFormat.textType) facets = r.to_facets() codes = r.to_enumeration(cls, valid) - cl_ref = r.representation ab = r.to_array_def() - return (dt, facets, codes, cl_ref, ab) + return (dt, facets, codes, ab) class FusionGroup(Struct, frozen=True): @@ -117,7 +115,7 @@ def to_model( ) -> Component: """Returns an attribute.""" c = _find_concept(cs, self.concept) - dt, facets, codes, cl_ref, ab = _get_representation( + dt, facets, codes, ab = _get_representation( self.id, self.representation, cls, cons, c ) lvl = self.__derive_level(groups) @@ -135,7 +133,6 @@ def to_model( desc, codes=codes, attachment_level=lvl, - enum_ref=cl_ref, array_def=ab, ) @@ -171,7 +168,7 @@ def to_model( ) -> Component: """Returns a dimension.""" c = _find_concept(cs, self.concept) - dt, facets, codes, cl_ref, ab = _get_representation( + dt, facets, codes, ab = _get_representation( self.id, self.representation, cls, cons, c ) if c.descriptions: @@ -187,7 +184,6 @@ def to_model( c.names[0].value, desc, codes=codes, - enum_ref=cl_ref, array_def=ab, ) @@ -223,7 +219,7 @@ def to_model( ) -> Component: """Returns a measure.""" c = _find_concept(cs, self.concept) - dt, facets, codes, cl_ref, ab = _get_representation( + dt, facets, codes, ab = _get_representation( self.id, self.representation, cls, cons, c ) if c.descriptions: @@ -239,7 +235,6 @@ def to_model( c.names[0].value, desc, codes=codes, - enum_ref=cl_ref, array_def=ab, ) diff --git a/src/pysdmx/fmr/sdmx/dsd.py b/src/pysdmx/fmr/sdmx/dsd.py index 98dd9b5..a8ed2c1 100644 --- a/src/pysdmx/fmr/sdmx/dsd.py +++ b/src/pysdmx/fmr/sdmx/dsd.py @@ -57,21 +57,18 @@ def _get_representation( codes: Sequence[Code] = [] dt = DataType.STRING facets = None - cl_ref = None ab = None if local: dt = DataType(__get_type(local)) facets = local.to_facets() codes = local.to_enumeration(cls, valid) - cl_ref = local.enumeration ab = local.to_array_def() elif core: dt = DataType(__get_type(core)) facets = core.to_facets() codes = core.to_enumeration(cls, valid) - cl_ref = core.enumeration ab = core.to_array_def() - return (dt, facets, codes, cl_ref, ab) + return (dt, facets, codes, ab) class JsonGroup(Struct, frozen=True): @@ -117,7 +114,7 @@ def to_model( ) -> Component: """Returns a component.""" c = _find_concept(cs, self.conceptIdentity) - dt, facets, codes, cl_ref, ab = _get_representation( + dt, facets, codes, ab = _get_representation( self.id, self.localRepresentation, c.coreRepresentation, cls, cons ) return Component( @@ -129,7 +126,6 @@ def to_model( c.name, c.description, codes=codes, - enum_ref=cl_ref, array_def=ab, ) @@ -153,7 +149,7 @@ def to_model( ) -> Component: """Returns a component.""" c = _find_concept(cs, self.conceptIdentity) - dt, facets, codes, cl_ref, ab = _get_representation( + dt, facets, codes, ab = _get_representation( self.id, self.localRepresentation, c.coreRepresentation, cls, cons ) req = self.usage != "optional" @@ -171,7 +167,6 @@ def to_model( c.description, codes=codes, attachment_level=lvl, - enum_ref=cl_ref, array_def=ab, ) @@ -192,7 +187,7 @@ def to_model( ) -> Component: """Returns a component.""" c = _find_concept(cs, self.conceptIdentity) - dt, facets, codes, cl_ref, ab = _get_representation( + dt, facets, codes, ab = _get_representation( self.id, self.localRepresentation, c.coreRepresentation, cls, cons ) req = self.usage != "optional" @@ -205,7 +200,6 @@ def to_model( c.name, c.description, codes=codes, - enum_ref=cl_ref, array_def=ab, ) diff --git a/src/pysdmx/model/dataflow.py b/src/pysdmx/model/dataflow.py index 1236e31..9bffbc1 100644 --- a/src/pysdmx/model/dataflow.py +++ b/src/pysdmx/model/dataflow.py @@ -95,8 +95,6 @@ class Component(Struct, frozen=True, omit_defaults=True): D (for dataset-level attributes), O (for observation-level attributes) or a combination of dimension IDs, separated by commas, for series- and group-level attributes). - enum_ref: The URN of the enumeration (codelist or valuelist) from - which the codes are taken. array_def: Any additional constraints for array types. """ @@ -109,7 +107,6 @@ class Component(Struct, frozen=True, omit_defaults=True): description: Optional[str] = None codes: Union[Codelist, Hierarchy, None] = None attachment_level: Optional[str] = None - enum_ref: Optional[str] = None array_def: Optional[ArrayBoundaries] = None def __str__(self) -> str: diff --git a/tests/fmr/schema_checks.py b/tests/fmr/schema_checks.py index 95aafad..97d3b44 100644 --- a/tests/fmr/schema_checks.py +++ b/tests/fmr/schema_checks.py @@ -59,14 +59,9 @@ async def check_coded_components(mock, fmr: AsyncRegistryClient, query, body): for comp in vc.components: if comp.id in exp: assert len(comp.codes) == exp.get(comp.id) - assert comp.enum_ref is not None - assert comp.enum_ref.startswith( - "urn:sdmx:org.sdmx.infomodel.codelist." - ) count += 1 else: - assert len(comp.codes) == 0 - assert comp.enum_ref is None + assert not comp.codes assert count == len(exp.keys()) diff --git a/tests/model/test_component.py b/tests/model/test_component.py index 06a6f1e..f693398 100644 --- a/tests/model/test_component.py +++ b/tests/model/test_component.py @@ -45,10 +45,9 @@ def test_defaults(fid, req, role): assert f.description is None assert not f.codes assert f.attachment_level is None - assert f.enum_ref is None -def test_full_initialization(fid, req, role, typ, cl_ref, array_def): +def test_full_initialization(fid, req, role, typ, array_def): facets = Facets(min_value=0, max_value="100") name = "Signal quality" desc = "The quality of the GPS signal" @@ -63,7 +62,6 @@ def test_full_initialization(fid, req, role, typ, cl_ref, array_def): name, desc, attachment_level=lvl, - enum_ref=cl_ref, array_def=array_def, ) @@ -76,7 +74,6 @@ def test_full_initialization(fid, req, role, typ, cl_ref, array_def): assert f.description == desc assert not f.codes assert f.attachment_level == lvl - assert f.enum_ref == cl_ref assert f.array_def == array_def From ee2d9a4c3b150a22562fae561af6ca24fc879a33 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 10:12:53 +0100 Subject: [PATCH 03/26] Added type of flat codelist --- src/pysdmx/model/code.py | 3 ++- tests/model/test_codelist.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/pysdmx/model/code.py b/src/pysdmx/model/code.py index 37f89a1..42ed886 100644 --- a/src/pysdmx/model/code.py +++ b/src/pysdmx/model/code.py @@ -17,7 +17,7 @@ """ from datetime import datetime -from typing import Iterator, Optional, Sequence +from typing import Iterator, Literal, Optional, Sequence from msgspec import Struct @@ -81,6 +81,7 @@ class Codelist(Struct, frozen=True, omit_defaults=True): description: Optional[str] = None version: str = "1.0" codes: Sequence[Code] = () + sdmx_type: Literal["codelist", "valuelist"] = "codelist" def __iter__(self) -> Iterator[Code]: """Return an iterator over the list of codes.""" diff --git a/tests/model/test_codelist.py b/tests/model/test_codelist.py index bd7f338..aacbd57 100644 --- a/tests/model/test_codelist.py +++ b/tests/model/test_codelist.py @@ -20,6 +20,11 @@ def agency(): return "5B0" +@pytest.fixture() +def sdmx_type(): + return "valuelist" + + def test_defaults(id, name, agency): cl = Codelist(id, name, agency) @@ -30,14 +35,15 @@ def test_defaults(id, name, agency): assert cl.version == "1.0" assert cl.codes is not None assert len(cl.codes) == 0 + assert cl.sdmx_type == "codelist" -def test_full_initialization(id, name, agency): +def test_full_initialization(id, name, agency, sdmx_type): desc = "description" version = "1.42.0" codes = [Code("child1", "Child 1"), Code("child2", "Child 2")] - cl = Codelist(id, name, agency, desc, version, codes) + cl = Codelist(id, name, agency, desc, version, codes, sdmx_type) assert cl.id == id assert cl.name == name @@ -47,6 +53,7 @@ def test_full_initialization(id, name, agency): assert cl.codes == codes assert len(cl) == 2 assert len(cl) == len(cl.codes) + assert cl.sdmx_type == sdmx_type def test_immutable(id, name, agency): From 02afd65bec607c79e9b7efc295813b562ca6d2e0 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 11:06:17 +0100 Subject: [PATCH 04/26] Component codes contain a codelist or a valuelist --- src/pysdmx/fmr/fusion/code.py | 3 +++ src/pysdmx/fmr/fusion/core.py | 26 +++++++++++++++----------- src/pysdmx/fmr/fusion/dsd.py | 12 +++--------- src/pysdmx/fmr/sdmx/core.py | 28 +++++++++++++++++----------- src/pysdmx/fmr/sdmx/dsd.py | 5 ++--- tests/fmr/code_checks.py | 1 + tests/fmr/schema_checks.py | 16 ++++++++++++++-- 7 files changed, 55 insertions(+), 36 deletions(-) diff --git a/src/pysdmx/fmr/fusion/code.py b/src/pysdmx/fmr/fusion/code.py index e7047f4..979cd5c 100644 --- a/src/pysdmx/fmr/fusion/code.py +++ b/src/pysdmx/fmr/fusion/code.py @@ -54,6 +54,7 @@ class FusionCodelist(Struct, frozen=True, rename={"agency": "agencyId"}): """Fusion-JSON payload for a codelist.""" id: str + urn: str names: Sequence[FusionString] agency: str descriptions: Sequence[FusionString] = () @@ -62,6 +63,7 @@ class FusionCodelist(Struct, frozen=True, rename={"agency": "agencyId"}): def to_model(self) -> CL: """Converts a JsonCodelist to a standard codelist.""" + t = "codelist" if "Codelist" in self.urn else "valuelist" return CL( self.id, self.names[0].value, @@ -69,6 +71,7 @@ def to_model(self) -> CL: self.descriptions[0].value if self.descriptions else None, self.version, [i.to_model() for i in self.items], + t, ) diff --git a/src/pysdmx/fmr/fusion/core.py b/src/pysdmx/fmr/fusion/core.py index 73c5c47..eca49c8 100644 --- a/src/pysdmx/fmr/fusion/core.py +++ b/src/pysdmx/fmr/fusion/core.py @@ -3,27 +3,27 @@ from datetime import datetime from typing import Any, Optional, Sequence, Union -from msgspec import Struct +import msgspec -from pysdmx.model import ArrayBoundaries, Code, Facets +from pysdmx.model import ArrayBoundaries, Codelist, Facets from pysdmx.util import find_by_urn -class FusionAnnotation(Struct, frozen=True): +class FusionAnnotation(msgspec.Struct, frozen=True): """Fusion-JSON payload for annotations.""" title: str type: str -class FusionString(Struct, frozen=True): +class FusionString(msgspec.Struct, frozen=True): """Fusion-JSON payload for an international string.""" locale: str value: str -class FusionTextFormat(Struct, frozen=True): +class FusionTextFormat(msgspec.Struct, frozen=True): """Fusion-JSON payload for TextFormat.""" textType: str @@ -40,7 +40,7 @@ class FusionTextFormat(Struct, frozen=True): isSequence: bool = False -class FusionRepresentation(Struct, frozen=True): +class FusionRepresentation(msgspec.Struct, frozen=True): """Fusion-JSON payload for core representation.""" textFormat: Optional[FusionTextFormat] = None @@ -83,13 +83,17 @@ def to_enumeration( self, codelists: Sequence[Any], valid: Sequence[str], - ) -> Sequence[Code]: + ) -> Optional[Codelist]: """Returns the list of codes allowed for this component.""" - codes = [] if self.representation: - a = find_by_urn(codelists, self.representation).items - codes = [c.to_model() for c in a if not valid or c.id in valid] - return codes + a = find_by_urn(codelists, self.representation) + if a: + cl = a.to_model() + codes = [ + c.to_model() for c in a.items if not valid or c.id in valid + ] + return msgspec.structs.replace(cl, codes=codes) + return None def to_array_def(self) -> Optional[ArrayBoundaries]: """Returns the array boundaries, if any.""" diff --git a/src/pysdmx/fmr/fusion/dsd.py b/src/pysdmx/fmr/fusion/dsd.py index 5341257..2fe2036 100644 --- a/src/pysdmx/fmr/fusion/dsd.py +++ b/src/pysdmx/fmr/fusion/dsd.py @@ -11,6 +11,7 @@ from pysdmx.model import ( ArrayBoundaries, Code, + Codelist, Component, Components, DataType, @@ -39,19 +40,12 @@ def _get_representation( cls: Sequence[FusionCodelist], cons: Dict[str, Sequence[str]], c: Optional[FusionConcept], -) -> Tuple[ - DataType, - Optional[Facets], - Sequence[Code], - Optional[str], - Optional[ArrayBoundaries], -]: +) -> Tuple[DataType, Optional[Facets], Codelist, Optional[ArrayBoundaries],]: valid = cons.get(id_, []) - codes: Sequence[Code] = [] ab = None dt = DataType.STRING facets = None - codes = [] + codes = None if repr_: r = repr_ elif c and c.representation: diff --git a/src/pysdmx/fmr/sdmx/core.py b/src/pysdmx/fmr/sdmx/core.py index 222f3a1..c6f1c4b 100644 --- a/src/pysdmx/fmr/sdmx/core.py +++ b/src/pysdmx/fmr/sdmx/core.py @@ -3,13 +3,13 @@ from datetime import datetime from typing import Optional, Sequence, Union -from msgspec import Struct +import msgspec from pysdmx.model import ArrayBoundaries, Code, Codelist, Facets from pysdmx.util import find_by_urn -class JsonAnnotation(Struct, frozen=True): +class JsonAnnotation(msgspec.Struct, frozen=True): """SDMX-JSON payload for annotations.""" title: str @@ -17,7 +17,7 @@ class JsonAnnotation(Struct, frozen=True): text: Optional[str] = None -class JsonTextFormat(Struct, frozen=True): +class JsonTextFormat(msgspec.Struct, frozen=True): """SDMX-JSON payload for TextFormat.""" textType: str @@ -34,7 +34,7 @@ class JsonTextFormat(Struct, frozen=True): isSequence: bool = False -class JsonRepresentation(Struct, frozen=True): +class JsonRepresentation(msgspec.Struct, frozen=True): """SDMX-JSON payload for core representation.""" enumerationFormat: Optional[JsonTextFormat] = None @@ -82,13 +82,19 @@ def to_enumeration( self, codelists: Sequence[Codelist], valid: Sequence[str], - ) -> Sequence[Code]: + ) -> Optional[Codelist]: """Returns the list of codes allowed for this component.""" - codes = [] if self.enumeration: - a = find_by_urn(codelists, self.enumeration).codes - codes = [c for c in a if not valid or c.id in valid] - return codes + a = find_by_urn(codelists, self.enumeration) + if a: + codes = [c for c in a.codes if not valid or c.id in valid] + clt = ( + "codelist" + if ".Codelist=" in self.enumeration + else "valuelist" + ) + return msgspec.structs.replace(a, codes=codes, sdmx_type=clt) + return None def to_array_def(self) -> Optional[ArrayBoundaries]: """Returns the array boundaries, if any.""" @@ -99,13 +105,13 @@ def to_array_def(self) -> Optional[ArrayBoundaries]: return None -class JsonLink(Struct, frozen=True): +class JsonLink(msgspec.Struct, frozen=True): """SDMX-JSON payload for link objects.""" urn: str -class JsonHeader(Struct, frozen=True): +class JsonHeader(msgspec.Struct, frozen=True): """SDMX-JSON payload for message header.""" links: Sequence[JsonLink] = () diff --git a/src/pysdmx/fmr/sdmx/dsd.py b/src/pysdmx/fmr/sdmx/dsd.py index a8ed2c1..34fabcb 100644 --- a/src/pysdmx/fmr/sdmx/dsd.py +++ b/src/pysdmx/fmr/sdmx/dsd.py @@ -49,12 +49,11 @@ def _get_representation( ) -> Tuple[ DataType, Optional[Facets], - Sequence[Code], - Optional[str], + Optional[Codelist], Optional[ArrayBoundaries], ]: valid = cons.get(id_, []) - codes: Sequence[Code] = [] + codes = None dt = DataType.STRING facets = None ab = None diff --git a/tests/fmr/code_checks.py b/tests/fmr/code_checks.py index 46f5289..e780188 100644 --- a/tests/fmr/code_checks.py +++ b/tests/fmr/code_checks.py @@ -24,6 +24,7 @@ def check_codelist(mock, fmr: RegistryClient, query, body): assert codelist.agency == "SDMX" assert codelist.description == "The frequency of the data" assert codelist.version == "2.0" + assert codelist.sdmx_type == "codelist" for code in codelist: assert isinstance(code, Code) diff --git a/tests/fmr/schema_checks.py b/tests/fmr/schema_checks.py index 97d3b44..83b6820 100644 --- a/tests/fmr/schema_checks.py +++ b/tests/fmr/schema_checks.py @@ -59,6 +59,15 @@ async def check_coded_components(mock, fmr: AsyncRegistryClient, query, body): for comp in vc.components: if comp.id in exp: assert len(comp.codes) == exp.get(comp.id) + assert comp.codes.id is not None + assert comp.codes.agency == "BIS" + assert comp.codes.version == "1.0" + assert comp.codes.name is not None + assert ( + comp.codes.sdmx_type == "valuelist" + if comp.id == "AVAILABILITY" + else "codelist" + ) count += 1 else: assert not comp.codes @@ -98,7 +107,10 @@ def check_unconstrained_coded_components( vc = fmr.get_schema("datastructure", "BIS", "BIS_CBS", "1.0") for comp in vc.components: - assert len(comp.codes) == exp.get(comp.id, 0) + if comp.id in exp: + assert len(comp.codes) == exp[comp.id] + else: + assert comp.codes is None def check_core_local_repr( @@ -128,7 +140,7 @@ def check_core_local_repr( assert freq.facets.max_length == 1 assert isinstance(title, Component) - assert len(title.codes) == 0 + assert not title.codes assert title.dtype == DataType.STRING assert title.facets is None From 4b31ae11cd16a8e0e538e4d1cf8193959fcbba5e Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 11:06:44 +0100 Subject: [PATCH 05/26] Remove unused import --- src/pysdmx/fmr/sdmx/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pysdmx/fmr/sdmx/core.py b/src/pysdmx/fmr/sdmx/core.py index c6f1c4b..6522c08 100644 --- a/src/pysdmx/fmr/sdmx/core.py +++ b/src/pysdmx/fmr/sdmx/core.py @@ -5,7 +5,7 @@ import msgspec -from pysdmx.model import ArrayBoundaries, Code, Codelist, Facets +from pysdmx.model import ArrayBoundaries, Codelist, Facets from pysdmx.util import find_by_urn From 7771dc9a32bf8b11c0b5d47c1e8c512daf7b5947 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 11:12:47 +0100 Subject: [PATCH 06/26] Add type when extracting valuelist --- src/pysdmx/fmr/sdmx/code.py | 1 + src/pysdmx/fmr/sdmx/core.py | 7 +------ tests/fmr/vl_code_checks.py | 1 + 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pysdmx/fmr/sdmx/code.py b/src/pysdmx/fmr/sdmx/code.py index d07b2b2..67fc4dd 100644 --- a/src/pysdmx/fmr/sdmx/code.py +++ b/src/pysdmx/fmr/sdmx/code.py @@ -80,6 +80,7 @@ def to_model(self) -> Codelist: self.description, self.version, [i.to_model() for i in self.valueItems], + "valuelist", ) diff --git a/src/pysdmx/fmr/sdmx/core.py b/src/pysdmx/fmr/sdmx/core.py index 6522c08..5ae1327 100644 --- a/src/pysdmx/fmr/sdmx/core.py +++ b/src/pysdmx/fmr/sdmx/core.py @@ -88,12 +88,7 @@ def to_enumeration( a = find_by_urn(codelists, self.enumeration) if a: codes = [c for c in a.codes if not valid or c.id in valid] - clt = ( - "codelist" - if ".Codelist=" in self.enumeration - else "valuelist" - ) - return msgspec.structs.replace(a, codes=codes, sdmx_type=clt) + return msgspec.structs.replace(a, codes=codes) return None def to_array_def(self) -> Optional[ArrayBoundaries]: diff --git a/tests/fmr/vl_code_checks.py b/tests/fmr/vl_code_checks.py index 0b0b411..5521ba1 100644 --- a/tests/fmr/vl_code_checks.py +++ b/tests/fmr/vl_code_checks.py @@ -18,6 +18,7 @@ def check_vl_codelist(mock, fmr: RegistryClient, q1, q2, body): assert codelist.agency == "TEST" assert codelist.description is None assert codelist.version == "1.0" + assert codelist.sdmx_type == "valuelist" for code in codelist: assert isinstance(code, Code) From 8872705a470128d3b499fe92ab9d6580fc76b8c7 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 11:17:45 +0100 Subject: [PATCH 07/26] Fix flake8 issues --- src/pysdmx/fmr/fusion/dsd.py | 1 - src/pysdmx/fmr/sdmx/dsd.py | 1 - src/pysdmx/model/dataflow.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pysdmx/fmr/fusion/dsd.py b/src/pysdmx/fmr/fusion/dsd.py index 2fe2036..f4ec45c 100644 --- a/src/pysdmx/fmr/fusion/dsd.py +++ b/src/pysdmx/fmr/fusion/dsd.py @@ -10,7 +10,6 @@ from pysdmx.fmr.fusion.core import FusionRepresentation, FusionString from pysdmx.model import ( ArrayBoundaries, - Code, Codelist, Component, Components, diff --git a/src/pysdmx/fmr/sdmx/dsd.py b/src/pysdmx/fmr/sdmx/dsd.py index 34fabcb..0d17ab4 100644 --- a/src/pysdmx/fmr/sdmx/dsd.py +++ b/src/pysdmx/fmr/sdmx/dsd.py @@ -9,7 +9,6 @@ from pysdmx.fmr.sdmx.core import JsonRepresentation from pysdmx.model import ( ArrayBoundaries, - Code, Codelist, Component, Components, diff --git a/src/pysdmx/model/dataflow.py b/src/pysdmx/model/dataflow.py index 9bffbc1..de45540 100644 --- a/src/pysdmx/model/dataflow.py +++ b/src/pysdmx/model/dataflow.py @@ -14,7 +14,7 @@ from msgspec import Struct -from pysdmx.model.code import Code, Codelist, Hierarchy +from pysdmx.model.code import Codelist, Hierarchy from pysdmx.model.concept import DataType, Facets from pysdmx.model.organisation import Organisation From 6771d8d15ff5db3cbb34b88a79ba99fa7217e516 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 11:23:43 +0100 Subject: [PATCH 08/26] Fix mypy issues --- src/pysdmx/fmr/fusion/code.py | 2 +- src/pysdmx/fmr/fusion/concept.py | 2 +- src/pysdmx/fmr/fusion/dsd.py | 7 ++++++- src/pysdmx/fmr/sdmx/concept.py | 2 +- src/pysdmx/model/concept.py | 4 ++-- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/pysdmx/fmr/fusion/code.py b/src/pysdmx/fmr/fusion/code.py index 979cd5c..849e1cf 100644 --- a/src/pysdmx/fmr/fusion/code.py +++ b/src/pysdmx/fmr/fusion/code.py @@ -71,7 +71,7 @@ def to_model(self) -> CL: self.descriptions[0].value if self.descriptions else None, self.version, [i.to_model() for i in self.items], - t, + t, # type: ignore[arg-type] ) diff --git a/src/pysdmx/fmr/fusion/concept.py b/src/pysdmx/fmr/fusion/concept.py index 7b40251..4962b15 100644 --- a/src/pysdmx/fmr/fusion/concept.py +++ b/src/pysdmx/fmr/fusion/concept.py @@ -28,7 +28,7 @@ def to_model(self, codelists: Sequence[FusionCodelist]) -> Concept: c = ( self.representation.to_enumeration(codelists, []) if self.representation - else [] + else None ) d = self.descriptions[0].value if self.descriptions else None cl_ref = ( diff --git a/src/pysdmx/fmr/fusion/dsd.py b/src/pysdmx/fmr/fusion/dsd.py index f4ec45c..b1d9b90 100644 --- a/src/pysdmx/fmr/fusion/dsd.py +++ b/src/pysdmx/fmr/fusion/dsd.py @@ -39,7 +39,12 @@ def _get_representation( cls: Sequence[FusionCodelist], cons: Dict[str, Sequence[str]], c: Optional[FusionConcept], -) -> Tuple[DataType, Optional[Facets], Codelist, Optional[ArrayBoundaries],]: +) -> Tuple[ + DataType, + Optional[Facets], + Optional[Codelist], + Optional[ArrayBoundaries], +]: valid = cons.get(id_, []) ab = None dt = DataType.STRING diff --git a/src/pysdmx/fmr/sdmx/concept.py b/src/pysdmx/fmr/sdmx/concept.py index 037dad5..7b3a9d3 100644 --- a/src/pysdmx/fmr/sdmx/concept.py +++ b/src/pysdmx/fmr/sdmx/concept.py @@ -37,7 +37,7 @@ def to_model(self, codelists: Sequence[JsonCodelist]) -> Concept: else: dt = DataType.STRING facets = None - codes = [] + codes = None cl_ref = None return Concept( self.id, diff --git a/src/pysdmx/model/concept.py b/src/pysdmx/model/concept.py index e6cd733..ff13023 100644 --- a/src/pysdmx/model/concept.py +++ b/src/pysdmx/model/concept.py @@ -16,7 +16,7 @@ from msgspec import Struct -from pysdmx.model.code import Code +from pysdmx.model.code import Codelist class DataType(str, Enum): @@ -149,7 +149,7 @@ class Concept(Struct, frozen=True, omit_defaults=True): facets: Optional[Facets] = None name: Optional[str] = None description: Optional[str] = None - codes: Sequence[Code] = () + codes: Optional[Codelist] = None enum_ref: Optional[str] = None def __str__(self) -> str: From eabc6233e531ce047887fcebeee50431d308829a Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 11:27:26 +0100 Subject: [PATCH 09/26] Fix concept tests --- tests/model/test_concept.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/model/test_concept.py b/tests/model/test_concept.py index f955768..8e3b560 100644 --- a/tests/model/test_concept.py +++ b/tests/model/test_concept.py @@ -21,7 +21,7 @@ def test_defaults(fid): assert f.facets is None assert f.name is None assert f.description is None - assert len(f.codes) == 0 + assert not f.codes assert f.enum_ref is None @@ -38,7 +38,7 @@ def test_full_initialization(fid): assert f.facets == facets assert f.name == name assert f.description == desc - assert len(f.codes) == 0 + assert not f.codes assert f.enum_ref is None From 37231343dcf2177ca1521605248ba191632002b2 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 11:32:18 +0100 Subject: [PATCH 10/26] Remove unnecessary check --- src/pysdmx/fmr/fusion/core.py | 11 +++++------ src/pysdmx/fmr/sdmx/core.py | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/pysdmx/fmr/fusion/core.py b/src/pysdmx/fmr/fusion/core.py index eca49c8..730d0ef 100644 --- a/src/pysdmx/fmr/fusion/core.py +++ b/src/pysdmx/fmr/fusion/core.py @@ -87,12 +87,11 @@ def to_enumeration( """Returns the list of codes allowed for this component.""" if self.representation: a = find_by_urn(codelists, self.representation) - if a: - cl = a.to_model() - codes = [ - c.to_model() for c in a.items if not valid or c.id in valid - ] - return msgspec.structs.replace(cl, codes=codes) + cl = a.to_model() + codes = [ + c.to_model() for c in a.items if not valid or c.id in valid + ] + return msgspec.structs.replace(cl, codes=codes) return None def to_array_def(self) -> Optional[ArrayBoundaries]: diff --git a/src/pysdmx/fmr/sdmx/core.py b/src/pysdmx/fmr/sdmx/core.py index 5ae1327..cb0d62b 100644 --- a/src/pysdmx/fmr/sdmx/core.py +++ b/src/pysdmx/fmr/sdmx/core.py @@ -86,9 +86,8 @@ def to_enumeration( """Returns the list of codes allowed for this component.""" if self.enumeration: a = find_by_urn(codelists, self.enumeration) - if a: - codes = [c for c in a.codes if not valid or c.id in valid] - return msgspec.structs.replace(a, codes=codes) + codes = [c for c in a.codes if not valid or c.id in valid] + return msgspec.structs.replace(a, codes=codes) return None def to_array_def(self) -> Optional[ArrayBoundaries]: From ad61e945b2c5f5956c7bd9ae0a214910264e4228 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 11:55:20 +0100 Subject: [PATCH 11/26] Add tests for codelist in concepts --- tests/fmr/concept_checks.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/fmr/concept_checks.py b/tests/fmr/concept_checks.py index 9b9b5d5..582b4f4 100644 --- a/tests/fmr/concept_checks.py +++ b/tests/fmr/concept_checks.py @@ -4,7 +4,7 @@ import pytest from pysdmx.fmr import AsyncRegistryClient, RegistryClient -from pysdmx.model import Code, Concept, ConceptScheme, DataType +from pysdmx.model import Code, Codelist, Concept, ConceptScheme, DataType def check_cs(mock, fmr: RegistryClient, query, body): @@ -58,6 +58,13 @@ def check_concept_details(mock, fmr: RegistryClient, query, body): assert concept.description is None assert concept.dtype == DataType.STRING assert len(concept.codes) == 3 + assert isinstance(concept.codes, Codelist) + assert concept.codes.id == "MEDAL_NMM" + assert concept.codes.agency == "BIS.MEDIT" + assert concept.codes.version == "1.0" + assert concept.codes.sdmx_type == "codelist" + assert concept.codes.name == "MEDAL Mapping Modes" + assert concept.codes.description is not None for c in concept.codes: isinstance(c, Code) assert c.id in ["F", "I", "W"] From 8d3728f545c2261c4a3ecb167fb5125a4e1a1cce Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 12:25:13 +0100 Subject: [PATCH 12/26] Bump version --- pyproject.toml | 2 +- src/pysdmx/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fd649aa..68ebdd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pysdmx" -version = "1.0.0-beta-6" +version = "1.0.0-beta-7" description = "Your opinionated Python SDMX library" authors = [ "Xavier Sosnovsky ", diff --git a/src/pysdmx/__init__.py b/src/pysdmx/__init__.py index 975be7e..774c2bf 100644 --- a/src/pysdmx/__init__.py +++ b/src/pysdmx/__init__.py @@ -1,3 +1,3 @@ """Your opinionated Python SDMX library.""" -__version__ = "1.0.0-beta-6" +__version__ = "1.0.0-beta-7" From 76ff8c0caf144e37654ea185d04f893110816514 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Thu, 8 Feb 2024 13:54:55 +0100 Subject: [PATCH 13/26] Add operator to hierarchy --- src/pysdmx/model/code.py | 8 ++++++++ tests/model/test_hierarchy.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/pysdmx/model/code.py b/src/pysdmx/model/code.py index 42ed886..65f3ad1 100644 --- a/src/pysdmx/model/code.py +++ b/src/pysdmx/model/code.py @@ -167,6 +167,13 @@ class Hierarchy(Struct, frozen=True, omit_defaults=True): respective composition"). version: The hierarchy version (e.g. 2.0.42) codes: The list of codes in the hierarchy. + operator: The URN of the operator to be applied to the items of an + hierarchy. This is mainly used for data validation or data + compilation purposes. For example, Let's assume a hierarchy with + a top level code (A), with 2 child codes (B and C). And let's + assume that the operator property references a VTL operator + representing a sum. This can then be used for validation purposes, + to check that A = B + C. """ id: str @@ -175,6 +182,7 @@ class Hierarchy(Struct, frozen=True, omit_defaults=True): description: Optional[str] = None version: str = "1.0" codes: Sequence[HierarchicalCode] = () + operator: Optional[str] = None def __iter__(self) -> Iterator[HierarchicalCode]: """Return an iterator over the list of codes.""" diff --git a/tests/model/test_hierarchy.py b/tests/model/test_hierarchy.py index 183d6db..5919ba0 100644 --- a/tests/model/test_hierarchy.py +++ b/tests/model/test_hierarchy.py @@ -20,6 +20,14 @@ def agency(): return "5B0" +@pytest.fixture() +def operator(): + return ( + "urn:sdmx:org.sdmx.infomodel.transformation." + "UserDefinedOperator=SDMX:OPS(1.0).SUM" + ) + + def test_defaults(id, name, agency): cs = Hierarchy(id, name, agency) @@ -30,9 +38,10 @@ def test_defaults(id, name, agency): assert cs.version == "1.0" assert cs.codes is not None assert len(cs.codes) == 0 + assert cs.operator is None -def test_full_initialization(id, name, agency): +def test_full_initialization(id, name, agency, operator): desc = "description" version = "1.42.0" grandchild = HierarchicalCode("Child211", "Child 2.1.1") @@ -52,6 +61,7 @@ def test_full_initialization(id, name, agency): assert cs.codes == codes assert len(cs) == 4 assert len(cs.codes) == 2 + assert cs.operator == operator def test_immutable(id, name, agency): From af327a38c295755e127764fc9ebd58a77b4623ea Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 09:29:31 +0100 Subject: [PATCH 14/26] Fix issue with hierarchy test --- tests/model/test_hierarchy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/model/test_hierarchy.py b/tests/model/test_hierarchy.py index 5919ba0..f941857 100644 --- a/tests/model/test_hierarchy.py +++ b/tests/model/test_hierarchy.py @@ -51,7 +51,7 @@ def test_full_initialization(id, name, agency, operator): HierarchicalCode("child2", "Child 2", codes=[child]), ] - cs = Hierarchy(id, name, agency, desc, version, codes) + cs = Hierarchy(id, name, agency, desc, version, codes, operator) assert cs.id == id assert cs.name == name From 4fe0b35a0925c68d0eed73ca402321218f7672e9 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 12:06:49 +0100 Subject: [PATCH 15/26] Add HierarchyAssociation to data model --- src/pysdmx/model/code.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/pysdmx/model/code.py b/src/pysdmx/model/code.py index 65f3ad1..e13ed5b 100644 --- a/src/pysdmx/model/code.py +++ b/src/pysdmx/model/code.py @@ -256,3 +256,17 @@ def by_id(self, id: str) -> Sequence[HierarchicalCode]: returned set. """ return self.__by_id(id, self.codes) + + +class HierarchyAssociation(Struct, frozen=True, omit_defaults=True): + """Links a hierarchy to a component withing the context of a dataflow.""" + + id: str + name: str + agency: str + hierarchy: Hierarchy + component_ref: str + context_ref: str + description: Optional[str] = None + version: str = "1.0" + operator: Optional[str] = None From cd00ebc51bb849158a901cff22c697c96d679281 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 12:07:26 +0100 Subject: [PATCH 16/26] Add tests for linking hierarchies and schemas --- tests/fmr/dataflow_checks.py | 16 +- tests/fmr/fusion/test_dataflows.py | 26 ++ tests/fmr/fusion/test_schemas.py | 169 ++++++++-- .../fmr/samples/df/hierarchy_hca.fusion.json | 176 ++++++++++ .../samples/df/hierarchy_schema.fusion.json | 314 ++++++++++++++++++ tests/fmr/samples/df/no_hca.fusion.json | 115 +++++++ tests/fmr/schema_checks.py | 134 +++++++- 7 files changed, 907 insertions(+), 43 deletions(-) create mode 100644 tests/fmr/samples/df/hierarchy_hca.fusion.json create mode 100644 tests/fmr/samples/df/hierarchy_schema.fusion.json create mode 100644 tests/fmr/samples/df/no_hca.fusion.json diff --git a/tests/fmr/dataflow_checks.py b/tests/fmr/dataflow_checks.py index 9b724ea..c2dc7b4 100644 --- a/tests/fmr/dataflow_checks.py +++ b/tests/fmr/dataflow_checks.py @@ -95,6 +95,8 @@ def check_dataflow_info_with_schema( schema_body, dataflow_query, dataflow_body, + hca_query, + hca_body, ): """get_schema() should return a schema.""" route1 = mock.get(schema_query).mock( @@ -103,6 +105,9 @@ def check_dataflow_info_with_schema( route2 = mock.get(dataflow_query).mock( return_value=httpx.Response(200, content=dataflow_body) ) + route3 = mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) dsi = fmr.get_dataflow_details( "BIS.CBS", "CBS", "1.0", DataflowDetails.SCHEMA @@ -110,6 +115,7 @@ def check_dataflow_info_with_schema( assert route1.called assert route2.called + assert route3.called __check_dsi_and_schema(dsi) @@ -120,19 +126,25 @@ async def check_async_dataflow_info( schema_body, dataflow_query, dataflow_body, + hca_query, + hca_body, ): """get_schema() should return a schema.""" - route1 = mock.get(schema_query).mock( - return_value=httpx.Response(200, content=schema_body) + route1 = mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) ) route2 = mock.get(dataflow_query).mock( return_value=httpx.Response(200, content=dataflow_body) ) + route3 = mock.get(schema_query).mock( + return_value=httpx.Response(200, content=schema_body) + ) dsi = await fmr.get_dataflow_details("BIS.CBS", "CBS", "1.0") assert route1.called assert route2.called + assert route3.called __check_dsi(dsi) diff --git a/tests/fmr/fusion/test_dataflows.py b/tests/fmr/fusion/test_dataflows.py index 0194613..d7b0f8b 100644 --- a/tests/fmr/fusion/test_dataflows.py +++ b/tests/fmr/fusion/test_dataflows.py @@ -38,6 +38,24 @@ def schema_query_no_version(fmr): return f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" +@pytest.fixture() +def no_hca_query(fmr): + res = "structure/dataflow/" + agency = "BIS.CBS" + id = "CBS" + version = "1.0" + return ( + f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + "?references=parentsandsiblings&detail=referencepartial" + ) + + +@pytest.fixture() +def no_hca_body(): + with open("tests/fmr/samples/df/no_hca.fusion.json", "rb") as f: + return f.read() + + @pytest.fixture() def dataflow_query(fmr): res = "structure/dataflow/" @@ -161,6 +179,8 @@ def test_returns_dataflow_info_with_schema( schema_body, core_dataflow_query, core_dataflow_body, + no_hca_query, + no_hca_body, ): """get_dataflow_details() should return information about a dataflow.""" checks.check_dataflow_info_with_schema( @@ -170,6 +190,8 @@ def test_returns_dataflow_info_with_schema( schema_body, core_dataflow_query, core_dataflow_body, + no_hca_query, + no_hca_body, ) @@ -181,6 +203,8 @@ async def test_async_returns_dataflow_info( schema_body, dataflow_query, dataflow_body, + no_hca_query, + no_hca_body, ): """get_dataflow_details() should return information about a dataflow.""" await checks.check_async_dataflow_info( @@ -190,6 +214,8 @@ async def test_async_returns_dataflow_info( schema_body, dataflow_query, dataflow_body, + no_hca_query, + no_hca_body, ) diff --git a/tests/fmr/fusion/test_schemas.py b/tests/fmr/fusion/test_schemas.py index 756fe88..06e277e 100644 --- a/tests/fmr/fusion/test_schemas.py +++ b/tests/fmr/fusion/test_schemas.py @@ -31,6 +31,39 @@ def query(fmr): return f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" +@pytest.fixture() +def no_hca_query(fmr): + res = "structure/dataflow/" + agency = "BIS.CBS" + id = "CBS" + version = "1.0" + return ( + f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + "?references=parentsandsiblings&detail=referencepartial" + ) + + +@pytest.fixture() +def hierarchy_hca_query(fmr): + res = "structure/dataflow/" + agency = "BIS" + id = "TEST_DF" + version = "1.0" + return ( + f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + "?references=parentsandsiblings&detail=referencepartial" + ) + + +@pytest.fixture() +def hierarchy_query(fmr): + res = "schema/dataflow/" + agency = "BIS" + id = "TEST_DF" + version = "1.0" + return f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + + @pytest.fixture() def no_const_query(fmr): res = "schema/datastructure/" @@ -70,61 +103,114 @@ def error_body(): return f.read() -def test_returns_validation_context(respx_mock, fmr, query, body): +@pytest.fixture() +def hierarchy_body(): + with open("tests/fmr/samples/df/hierarchy_schema.fusion.json", "rb") as f: + return f.read() + + +@pytest.fixture() +def hier_assoc_body(): + with open("tests/fmr/samples/df/hierarchy_hca.fusion.json", "rb") as f: + return f.read() + + +@pytest.fixture() +def no_hca_body(): + with open("tests/fmr/samples/df/no_hca.fusion.json", "rb") as f: + return f.read() + + +def test_returns_validation_context( + respx_mock, fmr, query, no_hca_query, body, no_hca_body +): """get_validation_context() should return a schema.""" - checks.check_schema(respx_mock, fmr, query, body) + checks.check_schema( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) @pytest.mark.asyncio() -async def test_codes(respx_mock, async_fmr, query, body): +async def test_codes( + respx_mock, async_fmr, query, no_hca_query, body, no_hca_body +): """Components have the expected number of codes.""" - await checks.check_coded_components(respx_mock, async_fmr, query, body) + await checks.check_coded_components( + respx_mock, async_fmr, query, no_hca_query, body, no_hca_body + ) -def test_codes_no_const(respx_mock, fmr, no_const_query, no_const_body): +def test_codes_no_const( + respx_mock, fmr, no_const_query, no_hca_query, no_const_body, no_hca_body +): """Components have the expected number of codes.""" checks.check_unconstrained_coded_components( - respx_mock, fmr, no_const_query, no_const_body + respx_mock, + fmr, + no_const_query, + no_hca_query, + no_const_body, + no_hca_body, ) -def test_core_local_repr(respx_mock, fmr, no_const_query, no_const_body): +def test_core_local_repr( + respx_mock, fmr, no_const_query, no_hca_query, no_const_body, no_hca_body +): """Components have the expected representation (local or core).""" checks.check_core_local_repr( respx_mock, fmr, no_const_query, + no_hca_query, no_const_body, + no_hca_body, ) -def test_roles(respx_mock, fmr, query, body): +def test_roles(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected role.""" - checks.check_roles(respx_mock, fmr, query, body) + checks.check_roles(respx_mock, fmr, query, no_hca_query, body, no_hca_body) -def test_types(respx_mock, fmr, query, body): +def test_types(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected type.""" - checks.check_types(respx_mock, fmr, query, body) + checks.check_types(respx_mock, fmr, query, no_hca_query, body, no_hca_body) -def test_facets(respx_mock, fmr, query, body): +def test_facets(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected facets.""" - checks.check_facets(respx_mock, fmr, query, body) + checks.check_facets( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_required(respx_mock, fmr, query, body): +def test_required(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected required flag.""" - checks.check_required(respx_mock, fmr, query, body) + checks.check_required( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_attachment_level(respx_mock, fmr, query, body): +def test_attachment_level( + respx_mock, fmr, query, no_hca_query, body, no_hca_body +): """Components have the expected attachment level.""" - checks.check_attachment_level(respx_mock, fmr, query, body) + checks.check_attachment_level( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_error_level(respx_mock, fmr, query, error_body): +def test_error_level( + respx_mock, fmr, query, no_hca_query, error_body, no_hca_body +): """Attachment level could not be inferred.""" + respx_mock.get(no_hca_query).mock( + return_value=httpx.Response( + 200, + content=no_hca_body, + ) + ) respx_mock.get(query).mock( return_value=httpx.Response( 200, @@ -144,21 +230,52 @@ def test_error_level(respx_mock, fmr, query, error_body): ) -def test_description(respx_mock, fmr, query, body): +def test_description(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected description.""" - checks.check_description(respx_mock, fmr, query, body) + checks.check_description( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_array_def(respx_mock, fmr, query, body): +def test_array_def(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Array components may have a min & max number of items.""" - checks.check_array_definition(respx_mock, fmr, query, body) + checks.check_array_definition( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_no_measure(respx_mock, fmr, query, no_measure_body): +def test_no_measure( + respx_mock, fmr, query, no_hca_query, no_measure_body, no_hca_body +): """DSD may not contain any measure.""" - checks.check_no_measure(respx_mock, fmr, query, no_measure_body) + checks.check_no_measure( + respx_mock, fmr, query, no_hca_query, no_measure_body, no_hca_body + ) -def test_no_attr(respx_mock, fmr, query, no_attr_body): +def test_no_attr( + respx_mock, fmr, query, no_hca_query, no_attr_body, no_hca_body +): """DSD may not contain any attribute.""" - checks.check_no_attrs(respx_mock, fmr, query, no_attr_body) + checks.check_no_attrs( + respx_mock, fmr, query, no_hca_query, no_attr_body, no_hca_body + ) + + +def test_has_hierarchy( + respx_mock, + fmr, + hierarchy_query, + hierarchy_hca_query, + hierarchy_body, + hier_assoc_body, +): + """Components may reference a hierarchy.""" + checks.check_hierarchy( + respx_mock, + fmr, + hierarchy_query, + hierarchy_hca_query, + hierarchy_body, + hier_assoc_body, + ) diff --git a/tests/fmr/samples/df/hierarchy_hca.fusion.json b/tests/fmr/samples/df/hierarchy_hca.fusion.json new file mode 100644 index 0000000..c72f3b4 --- /dev/null +++ b/tests/fmr/samples/df/hierarchy_hca.fusion.json @@ -0,0 +1,176 @@ +{ + "meta": { + "id": "IREF626949", + "test": false, + "prepared": "2024-02-09T07:11:31Z", + "contentLanguages": [ + "en" + ], + "sender": { + "id": "5B0" + } + }, + "Codelist": [ + { + "id": "CL_FREQ", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)", + "names": [ + { + "locale": "en", + "value": "Freq" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "isPartial": true, + "validityType": "standard", + "items": [ + { + "id": "M", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_FREQ(1.0).M", + "names": [ + { + "locale": "en", + "value": "Monthly" + } + ] + } + ] + }, + { + "id": "CL_OPTION_TYPE", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_OPTION_TYPE(1.0)", + "names": [ + { + "locale": "en", + "value": "Types of options" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "isPartial": false, + "validityType": "standard", + "items": [ + { + "id": "P", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).P", + "names": [ + { + "locale": "en", + "value": "Put" + } + ] + }, + { + "id": "C", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).C", + "names": [ + { + "locale": "en", + "value": "Call" + } + ] + }, + { + "id": "B", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).B", + "names": [ + { + "locale": "en", + "value": "Both" + } + ] + }, + { + "id": "X", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).X", + "names": [ + { + "locale": "en", + "value": "Unused" + } + ] + } + ] + } + ], + "Hierarchy": [ + { + "id": "H_OPTION_TYPE", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Hierarchy=BIS:H_OPTION_TYPE(1.0)", + "names": [ + { + "locale": "en", + "value": "Test Hierarchy" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "formalLevels": false, + "codes": [ + { + "id": "B", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.HierarchicalCode=BIS:H_OPTION_TYPE(1.0).B", + "code": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).B", + "codes": [ + { + "id": "C", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.HierarchicalCode=BIS:H_OPTION_TYPE(1.0).B.C", + "code": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).C" + }, + { + "id": "P", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.HierarchicalCode=BIS:H_OPTION_TYPE(1.0).B.P", + "code": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).P" + } + ] + } + ] + } + ], + "HierarchyAssociation": [ + { + "id": "HA_TEST", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.HierarchyAssociation=TEST:HA_TEST(1.0)", + "links": [ + { + "rel": "UserDefinedOperator", + "href": "https://mms-med-fmr-dev.apps.dev.ocp.bisinfo.org/sdmx/v2/structure/userdefinedoperatorscheme/SDMX/OPS/1.0/SUM", + "urn": "urn:sdmx:org.sdmx.infomodel.transformation.UserDefinedOperator=SDMX:OPS(1.0).SUM", + "type": "sdmx_artefact" + } + ], + "names": [ + { + "locale": "en", + "value": "Test Hierarchy Association" + } + ], + "agencyId": "TEST", + "version": "1.0", + "isFinal": false, + "hierarchyRef": "urn:sdmx:org.sdmx.infomodel.codelist.Hierarchy=BIS:H_OPTION_TYPE(1.0)", + "linkedStructureRef": "urn:sdmx:org.sdmx.infomodel.datastructure.Dimension=BIS:TEST(1.0).OPTION_TYPE", + "contextRef": "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS:DF_TEST(1.0)" + } + ], + "Dataflow": [ + { + "id": "DF_TEST", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS:DF_TEST(1.0)", + "names": [ + { + "locale": "en", + "value": "Test DF" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "dataStructureRef": "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=BIS:TEST(1.0)" + } + ] +} \ No newline at end of file diff --git a/tests/fmr/samples/df/hierarchy_schema.fusion.json b/tests/fmr/samples/df/hierarchy_schema.fusion.json new file mode 100644 index 0000000..7bfcac2 --- /dev/null +++ b/tests/fmr/samples/df/hierarchy_schema.fusion.json @@ -0,0 +1,314 @@ +{ + "meta": { + "id": "IREF638537", + "test": false, + "prepared": "2024-02-09T07:30:55Z", + "contentLanguages": [ + "en" + ], + "sender": { + "id": "FusionRegistry" + }, + "links": [ + { + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS:TEST(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=BIS:TEST(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.registry.DataConstraint=BIS:TEST(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.base.AgencyScheme=BIS:AGENCIES(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:OPTION_TYPE(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.ConceptScheme=BIS:CONCEPTS(1.0)" + } + ] + }, + "AgencyScheme": [ + { + "id": "AGENCIES", + "urn": "urn:sdmx:org.sdmx.infomodel.base.AgencyScheme=SDMX:AGENCIES(1.0)", + "names": [ + { + "locale": "en", + "value": "AGENCIES" + } + ], + "agencyId": "SDMX", + "version": "1.0", + "isFinal": false, + "isPartial": true, + "items": [ + { + "id": "BIS", + "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=SDMX:AGENCIES(1.0).BIS", + "names": [ + { + "locale": "en", + "value": "BIS" + } + ] + } + ] + } + ], + "Codelist": [ + { + "id": "CL_FREQ", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)", + "names": [ + { + "locale": "en", + "value": "Freq" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "isPartial": true, + "validityType": "standard", + "items": [ + { + "id": "M", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_FREQ(1.0).M", + "names": [ + { + "locale": "en", + "value": "Monthly" + } + ] + } + ] + }, + { + "id": "CL_OPTION_TYPE", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_OPTION_TYPE(1.0)", + "names": [ + { + "locale": "en", + "value": "Types of options" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "isPartial": false, + "validityType": "standard", + "items": [ + { + "id": "P", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).P", + "names": [ + { + "locale": "en", + "value": "Put" + } + ] + }, + { + "id": "C", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).C", + "names": [ + { + "locale": "en", + "value": "Call" + } + ] + }, + { + "id": "B", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).B", + "names": [ + { + "locale": "en", + "value": "Both" + } + ] + }, + { + "id": "X", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).X", + "names": [ + { + "locale": "en", + "value": "Unused" + } + ] + } + ] + } + ], + "ConceptScheme": [ + { + "id": "CONCEPTS", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.ConceptScheme=BIS:CONCEPTS(1.0)", + "names": [ + { + "locale": "en", + "value": "Concepts" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "isPartial": true, + "validityType": "standard", + "items": [ + { + "id": "FREQ", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).FREQ", + "names": [ + { + "locale": "en", + "value": "Frequency" + } + ], + "representation": { + "textFormat": { + "textType": "String" + }, + "representation": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" + } + }, + { + "id": "TIME_PERIOD", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).TIME_PERIOD", + "names": [ + { + "locale": "en", + "value": "Time Period" + } + ], + "representation": { + "textFormat": { + "textType": "ObservationalTimePeriod", + "sequence": false + } + } + }, + { + "id": "CONTRACT", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).CONTRACT", + "names": [ + { + "locale": "en", + "value": "Contract" + } + ] + }, + { + "id": "OBS_VALUE", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).OBS_VALUE", + "names": [ + { + "locale": "en", + "value": "The observation value" + } + ] + }, + { + "id": "OPTION_TYPE", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).OPTION_TYPE", + "names": [ + { + "locale": "en", + "value": "The type of option" + } + ], + "representation": { + "textFormat": { + "textType": "String" + }, + "representation": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:OPTION_TYPE(1.0)" + } + } + ] + } + ], + "DataStructure": [ + { + "id": "TEST", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=BIS:TEST(1.0)", + "names": [ + { + "locale": "en", + "value": "MDD Schema for OCC Post Trade Statistics" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "dimensionList": { + "dimensions": [ + { + "id": "FREQ", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dimension=BIS:TEST(1.0).FREQ", + "representation": { + "textFormat": { + "textType": "String" + }, + "representation": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" + }, + "concept": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).FREQ" + }, + { + "id": "CONTRACT", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dimension=BIS:TEST(1.0).CONTRACT", + "representation": { + "textFormat": { + "textType": "String" + } + }, + "concept": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).CONTRACT" + }, + { + "id": "OPTION_TYPE", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dimension=BIS:TEST(1.0).OPTION_TYPE", + "representation": { + "textFormat": { + "textType": "String" + }, + "representation": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_OPTION_TYPE(1.0)" + }, + "concept": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).OPTION_TYPE" + }, + { + "id": "TIME_PERIOD", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.TimeDimension=BIS:TEST(1.0).TIME_PERIOD", + "representation": { + "textFormat": { + "textType": "ObservationalTimePeriod" + } + }, + "concept": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).TIME_PERIOD", + "isTimeDimension": true + } + ] + }, + "measures": [ + { + "id": "OBS_VALUE", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Measure=BIS:TEST(1.0).OBS_VALUE", + "representation": { + "minOccurs": 1, + "textFormat": { + "textType": "BigInteger", + "sequence": false + } + }, + "concept": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).OBS_VALUE", + "mandatory": true + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/fmr/samples/df/no_hca.fusion.json b/tests/fmr/samples/df/no_hca.fusion.json new file mode 100644 index 0000000..7d4d8f8 --- /dev/null +++ b/tests/fmr/samples/df/no_hca.fusion.json @@ -0,0 +1,115 @@ +{ + "meta": { + "id": "IREF626949", + "test": false, + "prepared": "2024-02-09T07:11:31Z", + "contentLanguages": [ + "en" + ], + "sender": { + "id": "5B0" + } + }, + "Codelist": [ + { + "id": "CL_FREQ", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)", + "names": [ + { + "locale": "en", + "value": "Freq" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "isPartial": true, + "validityType": "standard", + "items": [ + { + "id": "M", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_FREQ(1.0).M", + "names": [ + { + "locale": "en", + "value": "Monthly" + } + ] + } + ] + }, + { + "id": "CL_OPTION_TYPE", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_OPTION_TYPE(1.0)", + "names": [ + { + "locale": "en", + "value": "Types of options" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "isPartial": false, + "validityType": "standard", + "items": [ + { + "id": "P", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).P", + "names": [ + { + "locale": "en", + "value": "Put" + } + ] + }, + { + "id": "C", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).C", + "names": [ + { + "locale": "en", + "value": "Call" + } + ] + }, + { + "id": "B", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).B", + "names": [ + { + "locale": "en", + "value": "Both" + } + ] + }, + { + "id": "X", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).X", + "names": [ + { + "locale": "en", + "value": "Unused" + } + ] + } + ] + } + ], + "Dataflow": [ + { + "id": "DF_TEST", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS:DF_TEST(1.0)", + "names": [ + { + "locale": "en", + "value": "Test DF" + } + ], + "agencyId": "BIS", + "version": "1.0", + "isFinal": false, + "dataStructureRef": "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=BIS:TEST(1.0)" + } + ] +} \ No newline at end of file diff --git a/tests/fmr/schema_checks.py b/tests/fmr/schema_checks.py index 83b6820..69304e1 100644 --- a/tests/fmr/schema_checks.py +++ b/tests/fmr/schema_checks.py @@ -3,11 +3,22 @@ import httpx from pysdmx.fmr import AsyncRegistryClient, RegistryClient -from pysdmx.model import Component, Components, DataType, Role, Schema - - -def check_schema(mock, fmr: RegistryClient, query, body): +from pysdmx.model import ( + Codelist, + Component, + Components, + DataType, + Hierarchy, + Role, + Schema, +) + + +def check_schema(mock, fmr: RegistryClient, query, hca_query, body, hca_body): """get_schema() should return a schema.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) vc = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0") @@ -27,8 +38,13 @@ def check_schema(mock, fmr: RegistryClient, query, body): assert comp.name is not None -async def check_coded_components(mock, fmr: AsyncRegistryClient, query, body): +async def check_coded_components( + mock, fmr: AsyncRegistryClient, query, hca_query, body, hca_body +): """Components have the expected number of codes.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) exp = { "FREQ": 1, @@ -75,9 +91,17 @@ async def check_coded_components(mock, fmr: AsyncRegistryClient, query, body): def check_unconstrained_coded_components( - mock, fmr: RegistryClient, no_const_query, no_const_body + mock, + fmr: RegistryClient, + no_const_query, + hca_query, + no_const_body, + hca_body, ): """Components have the expected number of codes.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(no_const_query).mock( return_value=httpx.Response(200, content=no_const_body) ) @@ -117,9 +141,14 @@ def check_core_local_repr( mock, fmr: RegistryClient, no_const_query, + hca_query, no_const_body, + hca_body, ): """Components have the expected representation (local or core).""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(no_const_query).mock( return_value=httpx.Response(200, content=no_const_body) ) @@ -145,8 +174,11 @@ def check_core_local_repr( assert title.facets is None -def check_roles(mock, fmr: RegistryClient, query, body): +def check_roles(mock, fmr: RegistryClient, query, hca_query, body, hca_body): """Components have the expected role.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) schema = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0").components @@ -160,8 +192,11 @@ def check_roles(mock, fmr: RegistryClient, query, body): assert schema["DECIMALS"] in schema.attributes -def check_types(mock, fmr: RegistryClient, query, body): +def check_types(mock, fmr: RegistryClient, query, hca_query, body, hca_body): """Components have the expected type.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) schema = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0").components @@ -175,8 +210,11 @@ def check_types(mock, fmr: RegistryClient, query, body): assert comp.dtype == DataType.STRING -def check_facets(mock, fmr: RegistryClient, query, body): +def check_facets(mock, fmr: RegistryClient, query, hca_query, body, hca_body): """Components have the expected facets.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) schema = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0").components @@ -195,8 +233,13 @@ def check_facets(mock, fmr: RegistryClient, query, body): assert comp.facets.max_length <= 200 -def check_required(mock, fmr: RegistryClient, query, body): +def check_required( + mock, fmr: RegistryClient, query, hca_query, body, hca_body +): """Components have the expected required flag.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) schema = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0").components @@ -213,8 +256,13 @@ def check_required(mock, fmr: RegistryClient, query, body): assert comp.required is True -def check_attachment_level(mock, fmr: RegistryClient, query, body): +def check_attachment_level( + mock, fmr: RegistryClient, query, hca_query, body, hca_body +): """Components have the expected attachment level.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) schema = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0").components @@ -231,8 +279,13 @@ def check_attachment_level(mock, fmr: RegistryClient, query, body): assert comp.attachment_level is None -def check_description(mock, fmr: RegistryClient, query, body): +def check_description( + mock, fmr: RegistryClient, query, hca_query, body, hca_body +): """Components may have a description.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) schema = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0").components @@ -248,8 +301,13 @@ def check_description(mock, fmr: RegistryClient, query, body): assert comp.description is None -def check_array_definition(mock, fmr: RegistryClient, query, body): +def check_array_definition( + mock, fmr: RegistryClient, query, hca_query, body, hca_body +): """Array components may have min & max number of items.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) schema = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0").components @@ -263,8 +321,13 @@ def check_array_definition(mock, fmr: RegistryClient, query, body): assert cmp.array_def is None -def check_no_measure(mock, fmr: RegistryClient, query, body): +def check_no_measure( + mock, fmr: RegistryClient, query, hca_query, body, hca_body +): """get_schema() should return a schema, possibly with no measure.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) vc = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0") @@ -281,8 +344,13 @@ def check_no_measure(mock, fmr: RegistryClient, query, body): assert len(vc.components.measures) == 0 -def check_no_attrs(mock, fmr: RegistryClient, query, body): +def check_no_attrs( + mock, fmr: RegistryClient, query, hca_query, body, hca_body +): """get_schema() should return a schema, possibly with no attributes.""" + mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) mock.get(query).mock(return_value=httpx.Response(200, content=body)) vc = fmr.get_schema("dataflow", "BIS.CBS", "CBS", "1.0") @@ -297,3 +365,39 @@ def check_no_attrs(mock, fmr: RegistryClient, query, body): assert isinstance(vc.components, Components) assert len(vc.components) == 13 assert len(vc.components.attributes) == 0 + + +def check_hierarchy( + mock, fmr: RegistryClient, query, query_hca, body, body_hca +): + """Some components may reference a hierarchy.""" + mock.get(query_hca).mock( + return_value=httpx.Response(200, content=body_hca) + ) + mock.get(query).mock(return_value=httpx.Response(200, content=body)) + + vc = fmr.get_schema("dataflow", "BIS", "TEST_DF", "1.0") + + assert isinstance(vc, Schema) + assert vc.agency == "BIS" + assert vc.id == "TEST_DF" + assert vc.version == "1.0" + assert vc.context == "dataflow" + assert len(vc.artefacts) == 8 + assert isinstance(vc.generated, datetime) + assert isinstance(vc.components, Components) + assert len(vc.components) == 5 + assert len(vc.components.attributes) == 0 + for d in vc.components.dimensions: + if d.id in ["CONTRACT", "TIME_PERIOD"]: + assert d.codes is None + elif d.id == "OPTION_TYPE": + assert isinstance(d.codes, Hierarchy) + assert len(d.codes) == 3 + assert d.codes.id == "H_OPTION_TYPE" + assert d.codes.operator == ( + "urn:sdmx:org.sdmx.infomodel.transformation." + "UserDefinedOperator=SDMX:OPS(1.0).SUM" + ) + else: + assert isinstance(d.codes, Codelist) From d940bf3fa6028df69614dfe803d8880c3dc990fd Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 12:07:35 +0100 Subject: [PATCH 17/26] Add HierarchyAssociation to data model --- src/pysdmx/model/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/pysdmx/model/__init__.py b/src/pysdmx/model/__init__.py index c91feea..2b62437 100644 --- a/src/pysdmx/model/__init__.py +++ b/src/pysdmx/model/__init__.py @@ -8,7 +8,13 @@ from typing import Any from pysdmx.model.category import Category, CategoryScheme -from pysdmx.model.code import Code, Codelist, HierarchicalCode, Hierarchy +from pysdmx.model.code import ( + Code, + Codelist, + HierarchicalCode, + Hierarchy, + HierarchyAssociation, +) from pysdmx.model.concept import Concept, ConceptScheme, DataType, Facets from pysdmx.model.dataflow import ( ArrayBoundaries, @@ -89,6 +95,7 @@ def encoders(obj: Any) -> Any: "Facets", "HierarchicalCode", "Hierarchy", + "HierarchyAssociation", "ImplicitComponentMap", "StructureMap", "MetadataAttribute", From 4160ccab5e8efa8e8a34756d8d13f2089392173d Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 12:08:09 +0100 Subject: [PATCH 18/26] Modify API to include hierarchy associations --- src/pysdmx/fmr/__init__.py | 55 ++++++++++++++++++++++++++------------ src/pysdmx/fmr/reader.py | 1 + 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/pysdmx/fmr/__init__.py b/src/pysdmx/fmr/__init__.py index 95a245d..d85c62c 100644 --- a/src/pysdmx/fmr/__init__.py +++ b/src/pysdmx/fmr/__init__.py @@ -1,6 +1,14 @@ """Retrieve metadata from an FMR instance.""" from enum import Enum -from typing import Any, Literal, NoReturn, Optional, Sequence, Tuple, Union +from typing import ( + Any, + Literal, + NoReturn, + Optional, + Sequence, + Tuple, + Union, +) import httpx from msgspec.json import decode @@ -15,6 +23,7 @@ ConceptScheme, DataflowInfo, Hierarchy, + HierarchyAssociation, MetadataReport, MultiRepresentationMap, Organisation, @@ -70,6 +79,10 @@ class Context(Enum): "structure/dataflow/{0}/{1}/{2}" "?detail=referencepartial&references={3}" ), + "ha": ( + "structure/dataflow/{0}/{1}/{2}" + "?references=parentsandsiblings&detail=referencepartial" + ), "hierarchy": ( "structure/hierarchy/{0}/{1}/{2}" "?detail=referencepartial&references=codelist" @@ -217,6 +230,12 @@ def __fetch(self, url: str, is_ref_meta: bool = False) -> bytes: except (httpx.RequestError, httpx.HTTPStatusError) as e: self._error(e) + def __get_hierarchies_for_flow( + self, agency: str, flow: str, version: str + ) -> Sequence[HierarchyAssociation]: + out = self.__fetch(super()._url("ha", agency, flow, version)) + return super()._out(out, self.deser.hier_assoc) + def get_agencies(self, agency: str) -> Sequence[Organisation]: """Get the list of **sub-agencies** for the supplied agency. @@ -339,16 +358,14 @@ def get_schema( Returns: The requested schema. """ + ha = ( + self.__get_hierarchies_for_flow(agency, id, version) + if context != "datastructure" + else () + ) c = context.value if isinstance(context, Context) else context out = self.__fetch(super()._url("schema", c, agency, id, version)) - return super()._out( - out, - self.deser.schema, - c, - agency, - id, - version, - ) + return super()._out(out, self.deser.schema, c, agency, id, version, ha) def get_dataflow_details( self, @@ -537,6 +554,12 @@ async def __fetch(self, url: str, is_ref_meta: bool = False) -> bytes: except (httpx.RequestError, httpx.HTTPStatusError) as e: self._error(e) + async def __get_hierarchies_for_flow( + self, agency: str, flow: str, version: str + ) -> Sequence[HierarchyAssociation]: + out = await self.__fetch(super()._url("ha", agency, flow, version)) + return super()._out(out, self.deser.hier_assoc) + async def get_agencies(self, agency: str) -> Sequence[Organisation]: """Get the list of **sub-agencies** for the supplied agency. @@ -657,16 +680,14 @@ async def get_schema( Returns: The requested schema. """ + ha = ( + await self.__get_hierarchies_for_flow(agency, id, version) + if context != "datastructure" + else () + ) c = context.value if isinstance(context, Context) else context r = await self.__fetch(super()._url("schema", c, agency, id, version)) - return super()._out( - r, - self.deser.schema, - c, - agency, - id, - version, - ) + return super()._out(r, self.deser.schema, c, agency, id, version, ha) async def get_dataflow_details( self, diff --git a/src/pysdmx/fmr/reader.py b/src/pysdmx/fmr/reader.py index 245b42b..57882c1 100644 --- a/src/pysdmx/fmr/reader.py +++ b/src/pysdmx/fmr/reader.py @@ -26,6 +26,7 @@ class Deserializers: dataflow: Deserializer providers: Deserializer schema: Deserializer + hier_assoc: Deserializer hierarchy: Deserializer report: Deserializer mapping: Deserializer From f907eaae9b7adec20fbfaf0468ec63d79db2d25e Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 12:08:41 +0100 Subject: [PATCH 19/26] Support for hierarchy association in fusion-json --- src/pysdmx/fmr/fusion/__init__.py | 2 ++ src/pysdmx/fmr/fusion/code.py | 56 +++++++++++++++++++++++++++++-- src/pysdmx/fmr/fusion/core.py | 6 ++++ src/pysdmx/fmr/fusion/schema.py | 34 +++++++++++-------- 4 files changed, 82 insertions(+), 16 deletions(-) diff --git a/src/pysdmx/fmr/fusion/__init__.py b/src/pysdmx/fmr/fusion/__init__.py index a62fc4a..861154d 100644 --- a/src/pysdmx/fmr/fusion/__init__.py +++ b/src/pysdmx/fmr/fusion/__init__.py @@ -3,6 +3,7 @@ from pysdmx.fmr.fusion.category import FusionCategorySchemeMessage from pysdmx.fmr.fusion.code import ( FusionCodelistMessage, + FusionHierarchyAssociationMessage, FusionHierarchyMessage, ) from pysdmx.fmr.fusion.concept import FusionConcepSchemeMessage @@ -24,6 +25,7 @@ dataflow=FusionDataflowMessage, # type: ignore[arg-type] providers=FusionProviderMessage, # type: ignore[arg-type] schema=FusionSchemaMessage, # type: ignore[arg-type] + hier_assoc=FusionHierarchyAssociationMessage, # type: ignore[arg-type] hierarchy=FusionHierarchyMessage, # type: ignore[arg-type] report=FusionMetadataMessage, # type: ignore[arg-type] mapping=FusionMappingMessage, # type: ignore[arg-type] diff --git a/src/pysdmx/fmr/fusion/code.py b/src/pysdmx/fmr/fusion/code.py index 849e1cf..8f6721a 100644 --- a/src/pysdmx/fmr/fusion/code.py +++ b/src/pysdmx/fmr/fusion/code.py @@ -5,14 +5,15 @@ from msgspec import Struct -from pysdmx.fmr.fusion.core import FusionAnnotation, FusionString +from pysdmx.fmr.fusion.core import FusionAnnotation, FusionLink, FusionString from pysdmx.model import ( Code, Codelist as CL, HierarchicalCode, Hierarchy as HCL, + HierarchyAssociation as HA, ) -from pysdmx.util import parse_item_urn +from pysdmx.util import find_by_urn, parse_item_urn class FusionCode(Struct, frozen=True): @@ -158,6 +159,42 @@ def to_model(self, codelists: Sequence[CL]) -> HCL: ) +class FusionHierarchyAssociation( + Struct, frozen=True, rename={"agency": "agencyId"} +): + """Fusion-JSON payload for a hierarchy.""" + + id: str + names: Sequence[FusionString] + agency: str + hierarchyRef: str + linkedStructureRef: str + contextRef: str + links: Sequence[FusionLink] + descriptions: Sequence[FusionString] = () + version: str = "1.0" + + def to_model( + self, + hierarchies: Sequence[FusionHierarchy], + codelists: Sequence[FusionCodelist], + ) -> HA: + """Converts a FusionHierarchyAssocation to a standard association.""" + cls = [cl.to_model() for cl in codelists] + m = find_by_urn(hierarchies, self.hierarchyRef).to_model(cls) + return HA( + self.id, + self.names[0].value, + self.agency, + m, + self.linkedStructureRef, + self.contextRef, + self.descriptions[0].value if self.descriptions else None, + self.version, + self.links[0].urn if self.links else None, + ) + + class FusionHierarchyMessage(Struct, frozen=True): """Fusion-JSON payload for /hierarchy queries.""" @@ -168,3 +205,18 @@ def to_model(self) -> HCL: """Returns the requested hierarchy.""" cls = [cl.to_model() for cl in self.Codelist] return self.Hierarchy[0].to_model(cls) + + +class FusionHierarchyAssociationMessage(Struct, frozen=True): + """Fusion-JSON payload for hierarchy associations.""" + + Codelist: Sequence[FusionCodelist] = () + Hierarchy: Sequence[FusionHierarchy] = () + HierarchyAssociation: Sequence[FusionHierarchyAssociation] = () + + def to_model(self) -> Sequence[HA]: + """Returns the requested hierarchy associations.""" + return [ + ha.to_model(self.Hierarchy, self.Codelist) + for ha in self.HierarchyAssociation + ] diff --git a/src/pysdmx/fmr/fusion/core.py b/src/pysdmx/fmr/fusion/core.py index 730d0ef..16ed70d 100644 --- a/src/pysdmx/fmr/fusion/core.py +++ b/src/pysdmx/fmr/fusion/core.py @@ -23,6 +23,12 @@ class FusionString(msgspec.Struct, frozen=True): value: str +class FusionLink(msgspec.Struct, frozen=True): + """Fusion-JSON payload for link objects.""" + + urn: str + + class FusionTextFormat(msgspec.Struct, frozen=True): """Fusion-JSON payload for TextFormat.""" diff --git a/src/pysdmx/fmr/fusion/schema.py b/src/pysdmx/fmr/fusion/schema.py index 64f9ef0..11f44dd 100644 --- a/src/pysdmx/fmr/fusion/schema.py +++ b/src/pysdmx/fmr/fusion/schema.py @@ -1,31 +1,24 @@ """Collection of Fusion-JSON schemas for SDMX-REST schema queries.""" from typing import List, Sequence -from msgspec import Struct +import msgspec from pysdmx.fmr.fusion.code import FusionCodelist from pysdmx.fmr.fusion.concept import FusionConceptScheme from pysdmx.fmr.fusion.constraint import FusionContentConstraint +from pysdmx.fmr.fusion.core import FusionLink from pysdmx.fmr.fusion.dsd import FusionDataStructure -from pysdmx.model import Schema +from pysdmx.model import Components, HierarchyAssociation, Schema +from pysdmx.util import parse_item_urn -class FusionLink(Struct, frozen=True): - """Fusion-JSON payload for link objects.""" - - urn: str - - -class FusionHeader(Struct, frozen=True): +class FusionHeader(msgspec.Struct, frozen=True): """Fusion-JSON payload for message header.""" links: Sequence[FusionLink] = () -class FusionSchemaMessage( - Struct, - frozen=True, -): +class FusionSchemaMessage(msgspec.Struct, frozen=True): """Fusion-JSON payload for /schema queries.""" meta: FusionHeader @@ -41,6 +34,7 @@ def to_model( agency: str, id_: str, version: str, + hierarchies: Sequence[HierarchyAssociation], ) -> Schema: """Returns the requested schema.""" cls: List[FusionCodelist] = [] @@ -49,5 +43,17 @@ def to_model( components = self.DataStructure[0].get_components( self.ConceptScheme, cls, self.DataConstraint ) + comp_dict = {c.id: c for c in components} urns = [a.urn for a in self.meta.links] - return Schema(context, agency, id_, components, version, urns) + for ha in hierarchies: + comp_id = parse_item_urn(ha.component_ref).item_id + h = msgspec.structs.replace(ha.hierarchy, operator=ha.operator) + comp_dict[comp_id] = msgspec.structs.replace( + components[comp_id], codes=h + ) + urns.append( + "urn:sdmx:org.sdmx.infomodel.codelist.Hierarchy=" + f"{h.agency}:{h.id}({h.version})" + ) + comps = Components(comp_dict.values()) + return Schema(context, agency, id_, comps, version, urns) From 80cbd2f830857bbd21e3d5e122a8bb3c07d8db84 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 13:33:38 +0100 Subject: [PATCH 20/26] Fix typo --- tests/fmr/samples/df/hierarchy_schema.fusion.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fmr/samples/df/hierarchy_schema.fusion.json b/tests/fmr/samples/df/hierarchy_schema.fusion.json index 7bfcac2..243d10d 100644 --- a/tests/fmr/samples/df/hierarchy_schema.fusion.json +++ b/tests/fmr/samples/df/hierarchy_schema.fusion.json @@ -20,13 +20,13 @@ "urn": "urn:sdmx:org.sdmx.infomodel.registry.DataConstraint=BIS:TEST(1.0)" }, { - "urn": "urn:sdmx:org.sdmx.infomodel.base.AgencyScheme=BIS:AGENCIES(1.0)" + "urn": "urn:sdmx:org.sdmx.infomodel.base.AgencyScheme=SDMX:AGENCIES(1.0)" }, { "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" }, { - "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:OPTION_TYPE(1.0)" + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_OPTION_TYPE(1.0)" }, { "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.ConceptScheme=BIS:CONCEPTS(1.0)" @@ -241,7 +241,7 @@ "names": [ { "locale": "en", - "value": "MDD Schema for OCC Post Trade Statistics" + "value": "Test" } ], "agencyId": "BIS", From dd5f1a1d6bcb29705a2ae98ac7e13f8407622b7a Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 13:42:24 +0100 Subject: [PATCH 21/26] Update docs --- src/pysdmx/fmr/fusion/code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pysdmx/fmr/fusion/code.py b/src/pysdmx/fmr/fusion/code.py index 8f6721a..cacff87 100644 --- a/src/pysdmx/fmr/fusion/code.py +++ b/src/pysdmx/fmr/fusion/code.py @@ -162,7 +162,7 @@ def to_model(self, codelists: Sequence[CL]) -> HCL: class FusionHierarchyAssociation( Struct, frozen=True, rename={"agency": "agencyId"} ): - """Fusion-JSON payload for a hierarchy.""" + """Fusion-JSON payload for a hierarchy association.""" id: str names: Sequence[FusionString] From 7220f6249aed564494af027b94e7201f7936a756 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 13:45:09 +0100 Subject: [PATCH 22/26] Links could be missing --- src/pysdmx/fmr/fusion/code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pysdmx/fmr/fusion/code.py b/src/pysdmx/fmr/fusion/code.py index cacff87..38e30bc 100644 --- a/src/pysdmx/fmr/fusion/code.py +++ b/src/pysdmx/fmr/fusion/code.py @@ -170,7 +170,7 @@ class FusionHierarchyAssociation( hierarchyRef: str linkedStructureRef: str contextRef: str - links: Sequence[FusionLink] + links: Sequence[FusionLink] = () descriptions: Sequence[FusionString] = () version: str = "1.0" From 8ef93aa21cc6df21bc2f7fe77d62fffb207cde05 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 15:01:50 +0100 Subject: [PATCH 23/26] Support for hierarchy association in sdmx-json --- src/pysdmx/fmr/sdmx/__init__.py | 7 ++- src/pysdmx/fmr/sdmx/code.py | 79 +++++++++++++++++++++++++++++++-- src/pysdmx/fmr/sdmx/core.py | 1 + src/pysdmx/fmr/sdmx/schema.py | 24 +++++++--- 4 files changed, 102 insertions(+), 9 deletions(-) diff --git a/src/pysdmx/fmr/sdmx/__init__.py b/src/pysdmx/fmr/sdmx/__init__.py index 478eda0..1f0f864 100644 --- a/src/pysdmx/fmr/sdmx/__init__.py +++ b/src/pysdmx/fmr/sdmx/__init__.py @@ -2,7 +2,11 @@ from pysdmx.fmr.reader import Deserializers from pysdmx.fmr.sdmx.category import JsonCategorySchemeMessage -from pysdmx.fmr.sdmx.code import JsonCodelistMessage, JsonHierarchyMessage +from pysdmx.fmr.sdmx.code import ( + JsonCodelistMessage, + JsonHierarchyAssociationMessage, + JsonHierarchyMessage, +) from pysdmx.fmr.sdmx.concept import JsonConcepSchemeMessage from pysdmx.fmr.sdmx.dataflow import JsonDataflowMessage from pysdmx.fmr.sdmx.map import ( @@ -21,6 +25,7 @@ dataflow=JsonDataflowMessage, # type: ignore[arg-type] providers=JsonProviderMessage, # type: ignore[arg-type] schema=JsonSchemaMessage, # type: ignore[arg-type] + hier_assoc=JsonHierarchyAssociationMessage, # type: ignore[arg-type] hierarchy=JsonHierarchyMessage, # type: ignore[arg-type] report=JsonMetadataMessage, # type: ignore[arg-type] mapping=JsonMappingMessage, # type: ignore[arg-type] diff --git a/src/pysdmx/fmr/sdmx/code.py b/src/pysdmx/fmr/sdmx/code.py index 67fc4dd..7827d53 100644 --- a/src/pysdmx/fmr/sdmx/code.py +++ b/src/pysdmx/fmr/sdmx/code.py @@ -5,9 +5,15 @@ from msgspec import Struct -from pysdmx.fmr.sdmx.core import JsonAnnotation -from pysdmx.model import Code, Codelist, HierarchicalCode, Hierarchy -from pysdmx.util import parse_item_urn +from pysdmx.fmr.sdmx.core import JsonAnnotation, JsonLink +from pysdmx.model import ( + Code, + Codelist, + HierarchicalCode, + Hierarchy, + HierarchyAssociation, +) +from pysdmx.util import find_by_urn, parse_item_urn class JsonCode(Struct, frozen=True): @@ -177,6 +183,48 @@ def to_model(self) -> Hierarchy: return self.hierarchies[0].to_model(cls) +class JsonHierarchyAssociation( + Struct, frozen=True, rename={"agency": "agencyID"} +): + """SDMX-JSON payload for a hierarchy association.""" + + id: str + name: str + agency: str + linkedHierarchy: str + linkedObject: str + contextObject: str + links: Sequence[JsonLink] = () + description: Optional[str] = None + version: str = "1.0" + + def to_model( + self, + hierarchies: Sequence[JsonHierarchy], + codelists: Sequence[JsonCodelist], + ) -> HierarchyAssociation: + """Converts a FusionHierarchyAssocation to a standard association.""" + cls = [cl.to_model() for cl in codelists] + m = find_by_urn(hierarchies, self.linkedHierarchy).to_model(cls) + lnk = list( + filter( + lambda i: hasattr(i, "rel") and i.rel == "UserDefinedOperator", + self.links, + ) + ) + return HierarchyAssociation( + self.id, + self.name, + self.agency, + m, + self.linkedObject, + self.contextObject, + self.description, + self.version, + lnk[0].urn if lnk else None, + ) + + class JsonHierarchyMessage(Struct, frozen=True): """SDMX-JSON payload for /hierarchy queries.""" @@ -185,3 +233,28 @@ class JsonHierarchyMessage(Struct, frozen=True): def to_model(self) -> Hierarchy: """Returns the requested hierarchy.""" return self.data.to_model() + + +class JsonHierarchyAssociations(Struct, frozen=True): + """SDMX-JSON payload for hierarchy associations.""" + + codelists: Sequence[JsonCodelist] = () + hierarchies: Sequence[JsonHierarchy] = () + hierarchyassociations: Sequence[JsonHierarchyAssociation] = () + + def to_model(self) -> Hierarchy: + """Returns the requested hierarchy associations.""" + return [ + ha.to_model(self.hierarchies, self.codelists) + for ha in self.hierarchyassociations + ] + + +class JsonHierarchyAssociationMessage(Struct, frozen=True): + """SDMX-JSON payload for hierarchy associations messages.""" + + data: JsonHierarchyAssociations + + def to_model(self) -> Hierarchy: + """Returns the requested hierarchy associations.""" + return self.data.to_model() diff --git a/src/pysdmx/fmr/sdmx/core.py b/src/pysdmx/fmr/sdmx/core.py index cb0d62b..1d6688c 100644 --- a/src/pysdmx/fmr/sdmx/core.py +++ b/src/pysdmx/fmr/sdmx/core.py @@ -103,6 +103,7 @@ class JsonLink(msgspec.Struct, frozen=True): """SDMX-JSON payload for link objects.""" urn: str + rel: Optional[str] = None class JsonHeader(msgspec.Struct, frozen=True): diff --git a/src/pysdmx/fmr/sdmx/schema.py b/src/pysdmx/fmr/sdmx/schema.py index 5c28c24..f0aa00f 100644 --- a/src/pysdmx/fmr/sdmx/schema.py +++ b/src/pysdmx/fmr/sdmx/schema.py @@ -1,18 +1,19 @@ """Collection of SDMX-JSON schemas for SDMX-REST schema queries.""" from typing import Sequence -from msgspec import Struct +import msgspec from pysdmx.fmr.sdmx.code import JsonCodelist, JsonValuelist from pysdmx.fmr.sdmx.concept import JsonConceptScheme from pysdmx.fmr.sdmx.constraint import JsonContentConstraint from pysdmx.fmr.sdmx.core import JsonHeader from pysdmx.fmr.sdmx.dsd import JsonDataStructure -from pysdmx.model import Components, Schema +from pysdmx.model import Components, HierarchyAssociation, Schema +from pysdmx.util import parse_item_urn class JsonSchemas( - Struct, + msgspec.Struct, frozen=True, ): """SDMX-JSON payload schema structures.""" @@ -33,7 +34,7 @@ def to_model(self) -> Components: class JsonSchemaMessage( - Struct, + msgspec.Struct, frozen=True, ): """SDMX-JSON payload for /schema queries.""" @@ -47,8 +48,21 @@ def to_model( agency: str, id_: str, version: str, + hierarchies: Sequence[HierarchyAssociation], ) -> Schema: """Returns the requested schema.""" components = self.data.to_model() + comp_dict = {c.id: c for c in components} urns = [a.urn for a in self.meta.links] - return Schema(context, agency, id_, components, version, urns) + for ha in hierarchies: + comp_id = parse_item_urn(ha.component_ref).item_id + h = msgspec.structs.replace(ha.hierarchy, operator=ha.operator) + comp_dict[comp_id] = msgspec.structs.replace( + components[comp_id], codes=h + ) + urns.append( + "urn:sdmx:org.sdmx.infomodel.codelist.Hierarchy=" + f"{h.agency}:{h.id}({h.version})" + ) + comps = Components(comp_dict.values()) + return Schema(context, agency, id_, comps, version, urns) From 4ca302c5b645b70e422288549c503b2aba8f22a1 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 15:09:00 +0100 Subject: [PATCH 24/26] Add tests for hierarchy assoc in SDMX-JSON --- tests/fmr/samples/df/hierarchy_hca.json | 243 +++++++++++ tests/fmr/samples/df/hierarchy_schema.json | 457 +++++++++++++++++++++ tests/fmr/samples/df/no_hca.json | 151 +++++++ tests/fmr/sdmx/test_dataflows.py | 26 ++ tests/fmr/sdmx/test_schemas.py | 159 +++++-- 5 files changed, 1011 insertions(+), 25 deletions(-) create mode 100644 tests/fmr/samples/df/hierarchy_hca.json create mode 100644 tests/fmr/samples/df/hierarchy_schema.json create mode 100644 tests/fmr/samples/df/no_hca.json diff --git a/tests/fmr/samples/df/hierarchy_hca.json b/tests/fmr/samples/df/hierarchy_hca.json new file mode 100644 index 0000000..ad3b827 --- /dev/null +++ b/tests/fmr/samples/df/hierarchy_hca.json @@ -0,0 +1,243 @@ +{ + "meta": { + "id": "IREF810806", + "test": false, + "schema": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "prepared": "2024-02-09T12:18:03Z", + "contentLanguages": [ + "en" + ], + "sender": { + "id": "5B0" + } + }, + "data": { + "codelists": [ + { + "links": [ + { + "rel": "self", + "type": "codelist", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" + } + ], + "id": "CL_FREQ", + "name": "cl", + "names": { + "en": "cl" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "isPartial": false, + "codes": [ + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_FREQ(1.0).M" + } + ], + "id": "M", + "annotations": [ + { + "id": "medal", + "title": "^((\\d{4})-(0[1-9]|1[0-2]))$", + "type": "format" + } + ], + "name": "Monthly", + "names": { + "en": "Monthly" + } + } + ] + }, + { + "links": [ + { + "rel": "self", + "type": "codelist", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS.XTD:CL_OPTION_TYPE(1.0)" + } + ], + "id": "CL_OPTION_TYPE", + "name": "Types of options", + "names": { + "en": "Types of options" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "isPartial": false, + "codes": [ + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).P" + } + ], + "id": "P", + "name": "Put", + "names": { + "en": "Put" + } + }, + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).C" + } + ], + "id": "C", + "name": "Call", + "names": { + "en": "Call" + } + }, + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).B" + } + ], + "id": "B", + "name": "Both", + "names": { + "en": "Both" + } + } + ] + } + ], + "hierarchies": [ + { + "links": [ + { + "rel": "self", + "type": "hierarchy", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Hierarchy=BIS:H_OPTION_TYPE(1.0)" + } + ], + "id": "H_OPTION_TYPE", + "name": "Test Hierarchy", + "names": { + "en": "Test Hierarchy" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "hasFormalLevels": false, + "hierarchicalCodes": [ + { + "links": [ + { + "rel": "self", + "type": "hierarchicalcode", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.HierarchicalCode=BIS:H_OPTION_TYPE(1.0).B" + } + ], + "id": "B", + "code": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).B", + "hierarchicalCodes": [ + { + "links": [ + { + "rel": "self", + "type": "hierarchicalcode", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.HierarchicalCode=BIS:H_OPTION_TYPE(1.0).B.C" + } + ], + "id": "AE", + "code": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).C" + }, + { + "links": [ + { + "rel": "self", + "type": "hierarchicalcode", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.HierarchicalCode=BIS:H_OPTION_TYPE(1.0).B.P" + } + ], + "id": "AR", + "code": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).P" + } + ] + } + ] + } + ], + "hierarchyassociations": [ + { + "links": [ + { + "rel": "self", + "type": "hierarchyassociation", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.HierarchyAssociation=TEST:HA_TEST(1.0)" + }, + { + "rel": "UserDefinedOperator", + "href": "https://mms-med-fmr-dev.apps.dev.ocp.bisinfo.org/sdmx/v2/structure/userdefinedoperatorscheme/SDMX/OPS/1.0/SUM", + "type": "sdmx_artefact", + "urn": "urn:sdmx:org.sdmx.infomodel.transformation.UserDefinedOperator=SDMX:OPS(1.0).SUM" + } + ], + "id": "HA_TEST", + "name": "Test Hierarchy Association", + "names": { + "en": "Test Hierarchy Association" + }, + "version": "1.0", + "agencyID": "TEST", + "isExternalReference": false, + "isFinal": false, + "linkedHierarchy": "urn:sdmx:org.sdmx.infomodel.codelist.Hierarchy=BIS:H_OPTION_TYPE(1.0)", + "linkedObject": "urn:sdmx:org.sdmx.infomodel.datastructure.Dimension=BIS:TEST(1.0).OPTION_TYPE", + "contextObject": "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS:DF_TEST(1.0)" + } + ], + "dataflows": [ + { + "links": [ + { + "rel": "self", + "type": "dataflow", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS:DF_TEST(1.0)" + } + ], + "id": "DF_TEST", + "name": "Tests", + "names": { + "en": "Tests" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "structure": "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=BIS:TEST(1.0)" + } + ] + } +} \ No newline at end of file diff --git a/tests/fmr/samples/df/hierarchy_schema.json b/tests/fmr/samples/df/hierarchy_schema.json new file mode 100644 index 0000000..809d6b5 --- /dev/null +++ b/tests/fmr/samples/df/hierarchy_schema.json @@ -0,0 +1,457 @@ +{ + "meta": { + "id": "IREF801714", + "test": false, + "schema": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "prepared": "2024-02-09T12:02:58Z", + "contentLanguages": [ + "en" + ], + "sender": { + "id": "FusionRegistry" + }, + "links": [ + { + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS.TEST(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=BIS.TEST(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.registry.DataConstraint=BIS.TEST(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.base.AgencyScheme=SDMX:AGENCIES(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_OPTION_TYPE(1.0)" + }, + { + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.ConceptScheme=BIS:CONCEPTS(1.0)" + } + ] + }, + "data": { + "agencySchemes": [ + { + "links": [ + { + "rel": "self", + "type": "agencyscheme", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.base.AgencyScheme=SDMX:AGENCIES(1.0)" + } + ], + "id": "AGENCIES", + "name": "AGENCIES", + "names": { + "en": "AGENCIES" + }, + "version": "1.0", + "agencyID": "SDMX", + "isExternalReference": false, + "isFinal": false, + "isPartial": true, + "agencies": [ + { + "links": [ + { + "rel": "self", + "type": "agency", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.base.Agency=SDMX:AGENCIES(1.0).BIS" + } + ], + "id": "BIS", + "name": "BIS", + "names": { + "en": "BIS" + } + } + ] + } + ], + "codelists": [ + { + "links": [ + { + "rel": "self", + "type": "codelist", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" + } + ], + "id": "CL_FREQ", + "name": "cl", + "names": { + "en": "cl" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "isPartial": true, + "codes": [ + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_FREQ(1.0).M" + } + ], + "id": "M", + "annotations": [ + { + "id": "medal", + "title": "^((\\d{4})-(0[1-9]|1[0-2]))$", + "type": "format" + } + ], + "name": "Monthly", + "names": { + "en": "Monthly" + } + } + ] + }, + { + "links": [ + { + "rel": "self", + "type": "codelist", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_OPTION_TYPE(1.0)" + } + ], + "id": "CL_OPTION_TYPE", + "name": "Types of options", + "names": { + "en": "Types of options" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "isPartial": false, + "codes": [ + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).P" + } + ], + "id": "P", + "name": "Put", + "names": { + "en": "Put" + } + }, + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).C" + } + ], + "id": "C", + "name": "Call", + "names": { + "en": "Call" + } + }, + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).B" + } + ], + "id": "B", + "name": "Both", + "names": { + "en": "Both" + } + } + ] + } + ], + "conceptSchemes": [ + { + "links": [ + { + "rel": "self", + "type": "conceptscheme", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.ConceptScheme=BIS:CONCEPTS(1.0)" + } + ], + "id": "CONCEPTS", + "name": "Concepts", + "names": { + "en": "Concepts" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "isPartial": true, + "concepts": [ + { + "links": [ + { + "rel": "self", + "type": "concept", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).FREQ" + } + ], + "id": "FREQ", + "name": "Frequency", + "names": { + "en": "Frequency" + }, + "coreRepresentation": { + "enumerationFormat": { + "textType": "String" + }, + "enumeration": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" + } + }, + { + "links": [ + { + "rel": "self", + "type": "concept", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).TIME_PERIOD" + } + ], + "id": "TIME_PERIOD", + "name": "Time Period", + "names": { + "en": "Time Period" + }, + "coreRepresentation": { + "format": { + "isSequence": false, + "textType": "ObservationalTimePeriod" + } + } + }, + { + "links": [ + { + "rel": "self", + "type": "concept", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).CONTRACT" + } + ], + "id": "CONTRACT", + "name": "Contract", + "names": { + "en": "Contract" + } + }, + { + "links": [ + { + "rel": "self", + "type": "concept", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).OBS_VALUE" + } + ], + "id": "OBS_VALUE", + "name": "The observation value", + "names": { + "en": "The observation value" + } + }, + { + "links": [ + { + "rel": "self", + "type": "concept", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:OCC_CONCEPTS(1.0).OPTION_TYPE" + } + ], + "id": "OPTION_TYPE", + "name": "The type of option", + "names": { + "en": "The type of option" + }, + "coreRepresentation": { + "enumerationFormat": { + "textType": "String" + }, + "enumeration": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL)OPTION_TYPE(1.0)" + } + } + ] + } + ], + "dataStructures": [ + { + "links": [ + { + "rel": "self", + "type": "datastructure", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=BIS.TEST(1.0)" + } + ], + "id": "TEST", + "name": "Test", + "names": { + "en": "Test" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "dataStructureComponents": { + "dimensionList": { + "links": [ + { + "rel": "self", + "type": "dimensiondescriptor", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.DimensionDescriptor=BIS.TEST(1.0).DimensionDescriptor" + } + ], + "id": "DimensionDescriptor", + "dimensions": [ + { + "position": 1, + "type": "Dimension", + "links": [ + { + "rel": "self", + "type": "dimension", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dimension=BIS.TEST(1.0).FREQ" + } + ], + "id": "FREQ", + "localRepresentation": { + "enumerationFormat": { + "textType": "String" + }, + "enumeration": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" + }, + "conceptIdentity": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).FREQ" + }, + { + "position": 2, + "type": "Dimension", + "links": [ + { + "rel": "self", + "type": "dimension", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dimension=BIS.TEST(1.0).CONTRACT" + } + ], + "id": "CONTRACT", + "localRepresentation": { + "enumerationFormat": { + "textType": "String" + } + }, + "conceptIdentity": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).CONTRACT" + }, + { + "position": 3, + "type": "Dimension", + "links": [ + { + "rel": "self", + "type": "dimension", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dimension=BIS.TEST(1.0).OPTION_TYPE" + } + ], + "id": "OPTION_TYPE", + "localRepresentation": { + "enumerationFormat": { + "textType": "String" + }, + "enumeration": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_OPTION_TYPE(1.0)" + }, + "conceptIdentity": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).OPTION_TYPE" + } + ], + "timeDimensions": [ + { + "position": 4, + "type": "TimeDimension", + "links": [ + { + "rel": "self", + "type": "timedimension", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.TimeDimension=BIS.TEST(1.0).TIME_PERIOD" + } + ], + "id": "TIME_PERIOD", + "localRepresentation": { + "format": { + "textType": "ObservationalTimePeriod" + } + }, + "conceptIdentity": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).TIME_PERIOD" + } + ] + }, + "measureList": { + "links": [ + { + "rel": "self", + "type": "measuredescriptor", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.MeasureDescriptor=BIS.TEST(1.0).MeasureDescriptor" + } + ], + "id": "MeasureDescriptor", + "measures": [ + { + "usage": "mandatory", + "links": [ + { + "rel": "self", + "type": "measure", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Measure=BIS.TEST(1.0).OBS_VALUE" + } + ], + "id": "OBS_VALUE", + "localRepresentation": { + "format": { + "isSequence": false, + "textType": "BigInteger" + }, + "minOccurs": 1 + }, + "conceptIdentity": "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept=BIS:CONCEPTS(1.0).OBS_VALUE" + } + ] + } + } + } + ] + } +} \ No newline at end of file diff --git a/tests/fmr/samples/df/no_hca.json b/tests/fmr/samples/df/no_hca.json new file mode 100644 index 0000000..f7c235c --- /dev/null +++ b/tests/fmr/samples/df/no_hca.json @@ -0,0 +1,151 @@ +{ + "meta": { + "id": "IREF810806", + "test": false, + "schema": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "prepared": "2024-02-09T12:18:03Z", + "contentLanguages": [ + "en" + ], + "sender": { + "id": "5B0" + } + }, + "data": { + "codelists": [ + { + "links": [ + { + "rel": "self", + "type": "codelist", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS:CL_FREQ(1.0)" + } + ], + "id": "CL_FREQ", + "name": "cl", + "names": { + "en": "cl" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "isPartial": false, + "codes": [ + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_FREQ(1.0).M" + } + ], + "id": "M", + "annotations": [ + { + "id": "medal", + "title": "^((\\d{4})-(0[1-9]|1[0-2]))$", + "type": "format" + } + ], + "name": "Monthly", + "names": { + "en": "Monthly" + } + } + ] + }, + { + "links": [ + { + "rel": "self", + "type": "codelist", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=BIS.XTD:CL_OPTION_TYPE(1.0)" + } + ], + "id": "CL_OPTION_TYPE", + "name": "Types of options", + "names": { + "en": "Types of options" + }, + "version": "1.0", + "agencyID": "BIS", + "isExternalReference": false, + "isFinal": false, + "isPartial": false, + "codes": [ + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).P" + } + ], + "id": "P", + "name": "Put", + "names": { + "en": "Put" + } + }, + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).C" + } + ], + "id": "C", + "name": "Call", + "names": { + "en": "Call" + } + }, + { + "links": [ + { + "rel": "self", + "type": "code", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.codelist.Code=BIS:CL_OPTION_TYPE(1.0).B" + } + ], + "id": "B", + "name": "Both", + "names": { + "en": "Both" + } + } + ] + } + ], + "dataflows": [ + { + "links": [ + { + "rel": "self", + "type": "dataflow", + "uri": "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/develop/structure-message/tools/schemas/2.0.0/sdmx-json-structure-schema.json", + "urn": "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=BIS:TEST_DF(1.0)" + } + ], + "id": "TEST_DF", + "name": "Tests", + "names": { + "en": "Tests" + }, + "version": "1.0", + "agencyID": "BIS.XTD", + "isExternalReference": false, + "isFinal": false, + "structure": "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=BIS:TEST(1.0)" + } + ] + } +} \ No newline at end of file diff --git a/tests/fmr/sdmx/test_dataflows.py b/tests/fmr/sdmx/test_dataflows.py index fc32ba6..6be76fc 100644 --- a/tests/fmr/sdmx/test_dataflows.py +++ b/tests/fmr/sdmx/test_dataflows.py @@ -38,6 +38,24 @@ def schema_query_no_version(fmr): return f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" +@pytest.fixture() +def no_hca_query(fmr): + res = "structure/dataflow/" + agency = "BIS.CBS" + id = "CBS" + version = "1.0" + return ( + f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + "?references=parentsandsiblings&detail=referencepartial" + ) + + +@pytest.fixture() +def no_hca_body(): + with open("tests/fmr/samples/df/no_hca.json", "rb") as f: + return f.read() + + @pytest.fixture() def dataflow_query(fmr): res = "structure/dataflow/" @@ -161,6 +179,8 @@ def test_returns_dataflow_info_with_schema( schema_body, core_dataflow_query, core_dataflow_body, + no_hca_query, + no_hca_body, ): """get_dataflow_details() should return information about a dataflow.""" checks.check_dataflow_info_with_schema( @@ -170,6 +190,8 @@ def test_returns_dataflow_info_with_schema( schema_body, core_dataflow_query, core_dataflow_body, + no_hca_query, + no_hca_body, ) @@ -181,6 +203,8 @@ async def test_async_returns_dataflow_info( schema_body, dataflow_query, dataflow_body, + no_hca_query, + no_hca_body, ): """get_dataflow_details() should return information about a dataflow.""" await checks.check_async_dataflow_info( @@ -190,6 +214,8 @@ async def test_async_returns_dataflow_info( schema_body, dataflow_query, dataflow_body, + no_hca_query, + no_hca_body, ) diff --git a/tests/fmr/sdmx/test_schemas.py b/tests/fmr/sdmx/test_schemas.py index a460bf6..adcadb0 100644 --- a/tests/fmr/sdmx/test_schemas.py +++ b/tests/fmr/sdmx/test_schemas.py @@ -29,6 +29,39 @@ def query(fmr): return f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" +@pytest.fixture() +def no_hca_query(fmr): + res = "structure/dataflow/" + agency = "BIS.CBS" + id = "CBS" + version = "1.0" + return ( + f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + "?references=parentsandsiblings&detail=referencepartial" + ) + + +@pytest.fixture() +def hierarchy_hca_query(fmr): + res = "structure/dataflow/" + agency = "BIS" + id = "TEST_DF" + version = "1.0" + return ( + f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + "?references=parentsandsiblings&detail=referencepartial" + ) + + +@pytest.fixture() +def hierarchy_query(fmr): + res = "schema/dataflow/" + agency = "BIS" + id = "TEST_DF" + version = "1.0" + return f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + + @pytest.fixture() def no_const_query(fmr): res = "schema/datastructure/" @@ -62,74 +95,150 @@ def no_attr_body(): return f.read() -def test_returns_validation_context(respx_mock, fmr, query, body): +@pytest.fixture() +def hierarchy_body(): + with open("tests/fmr/samples/df/hierarchy_schema.json", "rb") as f: + return f.read() + + +@pytest.fixture() +def hier_assoc_body(): + with open("tests/fmr/samples/df/hierarchy_hca.json", "rb") as f: + return f.read() + + +@pytest.fixture() +def no_hca_body(): + with open("tests/fmr/samples/df/no_hca.json", "rb") as f: + return f.read() + + +def test_returns_validation_context( + respx_mock, fmr, query, no_hca_query, body, no_hca_body +): """get_validation_context() should return a schema.""" - checks.check_schema(respx_mock, fmr, query, body) + checks.check_schema( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) @pytest.mark.asyncio() -async def test_codes(respx_mock, async_fmr, query, body): +async def test_codes( + respx_mock, async_fmr, query, no_hca_query, body, no_hca_body +): """Components have the expected number of codes.""" - await checks.check_coded_components(respx_mock, async_fmr, query, body) + await checks.check_coded_components( + respx_mock, async_fmr, query, no_hca_query, body, no_hca_body + ) -def test_codes_no_const(respx_mock, fmr, no_const_query, no_const_body): +def test_codes_no_const( + respx_mock, fmr, no_const_query, no_hca_query, no_const_body, no_hca_body +): """Components have the expected number of codes.""" checks.check_unconstrained_coded_components( - respx_mock, fmr, no_const_query, no_const_body + respx_mock, + fmr, + no_const_query, + no_hca_query, + no_const_body, + no_hca_body, ) -def test_core_local_repr(respx_mock, fmr, no_const_query, no_const_body): +def test_core_local_repr( + respx_mock, fmr, no_const_query, no_hca_query, no_const_body, no_hca_body +): """Components have the expected representation (local or core).""" checks.check_core_local_repr( respx_mock, fmr, no_const_query, + no_hca_query, no_const_body, + no_hca_body, ) -def test_roles(respx_mock, fmr, query, body): +def test_roles(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected role.""" - checks.check_roles(respx_mock, fmr, query, body) + checks.check_roles(respx_mock, fmr, query, no_hca_query, body, no_hca_body) -def test_types(respx_mock, fmr, query, body): +def test_types(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected type.""" - checks.check_types(respx_mock, fmr, query, body) + checks.check_types(respx_mock, fmr, query, no_hca_query, body, no_hca_body) -def test_facets(respx_mock, fmr, query, body): +def test_facets(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected facets.""" - checks.check_facets(respx_mock, fmr, query, body) + checks.check_facets( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_required(respx_mock, fmr, query, body): +def test_required(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected required flag.""" - checks.check_required(respx_mock, fmr, query, body) + checks.check_required( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_attachment_level(respx_mock, fmr, query, body): +def test_attachment_level( + respx_mock, fmr, query, no_hca_query, body, no_hca_body +): """Components have the expected attachment level.""" - checks.check_attachment_level(respx_mock, fmr, query, body) + checks.check_attachment_level( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_description(respx_mock, fmr, query, body): +def test_description(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Components have the expected description.""" - checks.check_description(respx_mock, fmr, query, body) + checks.check_description( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_array_def(respx_mock, fmr, query, body): +def test_array_def(respx_mock, fmr, query, no_hca_query, body, no_hca_body): """Array components may have a min & max number of items.""" - checks.check_array_definition(respx_mock, fmr, query, body) + checks.check_array_definition( + respx_mock, fmr, query, no_hca_query, body, no_hca_body + ) -def test_no_measure(respx_mock, fmr, query, no_measure_body): +def test_no_measure( + respx_mock, fmr, query, no_hca_query, no_measure_body, no_hca_body +): """DSD may not contain any measure.""" - checks.check_no_measure(respx_mock, fmr, query, no_measure_body) + checks.check_no_measure( + respx_mock, fmr, query, no_hca_query, no_measure_body, no_hca_body + ) -def test_no_attr(respx_mock, fmr, query, no_attr_body): +def test_no_attr( + respx_mock, fmr, query, no_hca_query, no_attr_body, no_hca_body +): """DSD may not contain any attribute.""" - checks.check_no_attrs(respx_mock, fmr, query, no_attr_body) + checks.check_no_attrs( + respx_mock, fmr, query, no_hca_query, no_attr_body, no_hca_body + ) + + +def test_has_hierarchy( + respx_mock, + fmr, + hierarchy_query, + hierarchy_hca_query, + hierarchy_body, + hier_assoc_body, +): + """Components may reference a hierarchy.""" + checks.check_hierarchy( + respx_mock, + fmr, + hierarchy_query, + hierarchy_hca_query, + hierarchy_body, + hier_assoc_body, + ) From 8f656d581e41033b4a095ae134db4c858fa6c2ee Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 15:26:48 +0100 Subject: [PATCH 25/26] All refs are needed for hierarchy assoc queries --- src/pysdmx/fmr/__init__.py | 2 +- tests/fmr/dataflow_checks.py | 20 ++++++++++++++++---- tests/fmr/fusion/test_dataflows.py | 22 +++++++++++++++++++++- tests/fmr/fusion/test_schemas.py | 4 ++-- tests/fmr/sdmx/test_dataflows.py | 22 +++++++++++++++++++++- tests/fmr/sdmx/test_schemas.py | 4 ++-- 6 files changed, 63 insertions(+), 11 deletions(-) diff --git a/src/pysdmx/fmr/__init__.py b/src/pysdmx/fmr/__init__.py index d85c62c..6d27661 100644 --- a/src/pysdmx/fmr/__init__.py +++ b/src/pysdmx/fmr/__init__.py @@ -81,7 +81,7 @@ class Context(Enum): ), "ha": ( "structure/dataflow/{0}/{1}/{2}" - "?references=parentsandsiblings&detail=referencepartial" + "?references=all&detail=referencepartial" ), "hierarchy": ( "structure/hierarchy/{0}/{1}/{2}" diff --git a/tests/fmr/dataflow_checks.py b/tests/fmr/dataflow_checks.py index c2dc7b4..f3c0c2b 100644 --- a/tests/fmr/dataflow_checks.py +++ b/tests/fmr/dataflow_checks.py @@ -11,19 +11,25 @@ def check_dataflow_info( schema_body, dataflow_query, dataflow_body, + hca_query, + hca_body, ): """get_schema() should return a schema.""" - route1 = mock.get(schema_query).mock( - return_value=httpx.Response(200, content=schema_body) + route1 = mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) ) route2 = mock.get(dataflow_query).mock( return_value=httpx.Response(200, content=dataflow_body) ) + route3 = mock.get(schema_query).mock( + return_value=httpx.Response(200, content=schema_body) + ) dsi = fmr.get_dataflow_details("BIS.CBS", "CBS", "1.0") assert route1.called assert route2.called + assert route3.called __check_dsi(dsi) @@ -34,12 +40,17 @@ def check_dataflow_info_no_version( schema_body, dataflow_query_no_version, dataflow_body, + hca_query, + hca_body, ): """get_schema() should return a schema.""" - route1 = mock.get(schema_query).mock( + route1 = mock.get(hca_query).mock( + return_value=httpx.Response(200, content=hca_body) + ) + route2 = mock.get(schema_query).mock( return_value=httpx.Response(200, content=schema_body) ) - route2 = mock.get(dataflow_query_no_version).mock( + route3 = mock.get(dataflow_query_no_version).mock( return_value=httpx.Response(200, content=dataflow_body) ) @@ -47,6 +58,7 @@ def check_dataflow_info_no_version( assert route1.called assert route2.called + assert route3.called __check_dsi(dsi) diff --git a/tests/fmr/fusion/test_dataflows.py b/tests/fmr/fusion/test_dataflows.py index d7b0f8b..47ac2f4 100644 --- a/tests/fmr/fusion/test_dataflows.py +++ b/tests/fmr/fusion/test_dataflows.py @@ -46,7 +46,19 @@ def no_hca_query(fmr): version = "1.0" return ( f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" - "?references=parentsandsiblings&detail=referencepartial" + "?references=all&detail=referencepartial" + ) + + +@pytest.fixture() +def no_hca_query_no_version(fmr): + res = "structure/dataflow/" + agency = "BIS.CBS" + id = "CBS" + version = "+" + return ( + f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + "?references=all&detail=referencepartial" ) @@ -111,6 +123,8 @@ def test_returns_dataflow_info( schema_body, dataflow_query, dataflow_body, + no_hca_query, + no_hca_body, ): """get_dataflow_details() should return information about a dataflow.""" checks.check_dataflow_info( @@ -120,6 +134,8 @@ def test_returns_dataflow_info( schema_body, dataflow_query, dataflow_body, + no_hca_query, + no_hca_body, ) @@ -130,6 +146,8 @@ def test_returns_dataflow_no_version( schema_body, dataflow_query_no_version, dataflow_body, + no_hca_query_no_version, + no_hca_body, ): """get_dataflow_details() return information about a dataflow (+).""" checks.check_dataflow_info_no_version( @@ -139,6 +157,8 @@ def test_returns_dataflow_no_version( schema_body, dataflow_query_no_version, dataflow_body, + no_hca_query_no_version, + no_hca_body, ) diff --git a/tests/fmr/fusion/test_schemas.py b/tests/fmr/fusion/test_schemas.py index 06e277e..3b19378 100644 --- a/tests/fmr/fusion/test_schemas.py +++ b/tests/fmr/fusion/test_schemas.py @@ -39,7 +39,7 @@ def no_hca_query(fmr): version = "1.0" return ( f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" - "?references=parentsandsiblings&detail=referencepartial" + "?references=all&detail=referencepartial" ) @@ -51,7 +51,7 @@ def hierarchy_hca_query(fmr): version = "1.0" return ( f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" - "?references=parentsandsiblings&detail=referencepartial" + "?references=all&detail=referencepartial" ) diff --git a/tests/fmr/sdmx/test_dataflows.py b/tests/fmr/sdmx/test_dataflows.py index 6be76fc..66b5286 100644 --- a/tests/fmr/sdmx/test_dataflows.py +++ b/tests/fmr/sdmx/test_dataflows.py @@ -46,7 +46,19 @@ def no_hca_query(fmr): version = "1.0" return ( f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" - "?references=parentsandsiblings&detail=referencepartial" + "?references=all&detail=referencepartial" + ) + + +@pytest.fixture() +def no_hca_query_no_version(fmr): + res = "structure/dataflow/" + agency = "BIS.CBS" + id = "CBS" + version = "+" + return ( + f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" + "?references=all&detail=referencepartial" ) @@ -111,6 +123,8 @@ def test_returns_dataflow_info( schema_body, dataflow_query, dataflow_body, + no_hca_query, + no_hca_body, ): """get_dataflow_details() should return information about a dataflow.""" checks.check_dataflow_info( @@ -120,6 +134,8 @@ def test_returns_dataflow_info( schema_body, dataflow_query, dataflow_body, + no_hca_query, + no_hca_body, ) @@ -130,6 +146,8 @@ def test_returns_dataflow_no_version( schema_body, dataflow_query_no_version, dataflow_body, + no_hca_query_no_version, + no_hca_body, ): """get_dataflow_details() return information about a dataflow (+).""" checks.check_dataflow_info_no_version( @@ -139,6 +157,8 @@ def test_returns_dataflow_no_version( schema_body, dataflow_query_no_version, dataflow_body, + no_hca_query_no_version, + no_hca_body, ) diff --git a/tests/fmr/sdmx/test_schemas.py b/tests/fmr/sdmx/test_schemas.py index adcadb0..c96b262 100644 --- a/tests/fmr/sdmx/test_schemas.py +++ b/tests/fmr/sdmx/test_schemas.py @@ -37,7 +37,7 @@ def no_hca_query(fmr): version = "1.0" return ( f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" - "?references=parentsandsiblings&detail=referencepartial" + "?references=all&detail=referencepartial" ) @@ -49,7 +49,7 @@ def hierarchy_hca_query(fmr): version = "1.0" return ( f"{fmr.api_endpoint}{res}{agency}/{id}/{version}" - "?references=parentsandsiblings&detail=referencepartial" + "?references=all&detail=referencepartial" ) From 2fc378d1f3fa7827df13c7382f581aa34942beea Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 9 Feb 2024 15:30:23 +0100 Subject: [PATCH 26/26] Fix mypy issues --- src/pysdmx/fmr/sdmx/code.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pysdmx/fmr/sdmx/code.py b/src/pysdmx/fmr/sdmx/code.py index 7827d53..b0dcec8 100644 --- a/src/pysdmx/fmr/sdmx/code.py +++ b/src/pysdmx/fmr/sdmx/code.py @@ -242,7 +242,7 @@ class JsonHierarchyAssociations(Struct, frozen=True): hierarchies: Sequence[JsonHierarchy] = () hierarchyassociations: Sequence[JsonHierarchyAssociation] = () - def to_model(self) -> Hierarchy: + def to_model(self) -> Sequence[HierarchyAssociation]: """Returns the requested hierarchy associations.""" return [ ha.to_model(self.hierarchies, self.codelists) @@ -255,6 +255,6 @@ class JsonHierarchyAssociationMessage(Struct, frozen=True): data: JsonHierarchyAssociations - def to_model(self) -> Hierarchy: + def to_model(self) -> Sequence[HierarchyAssociation]: """Returns the requested hierarchy associations.""" return self.data.to_model()