From 04c536d01c3fe67a56bd6387f5e133a236b6cb9a Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 13:37:22 +0200 Subject: [PATCH 01/12] Add dataflowref, data consumer and provider --- src/pysdmx/model/__base.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/pysdmx/model/__base.py b/src/pysdmx/model/__base.py index e2323d4..c02a570 100644 --- a/src/pysdmx/model/__base.py +++ b/src/pysdmx/model/__base.py @@ -159,13 +159,20 @@ class Organisation(Item, frozen=True, omit_defaults=True): Attributes: contacts: The contact of the agency. + dataflows: The dataflows relevant for the organisation. For example, + the list of dataflows for which a data provider provides data. """ contacts: Sequence[Contact] = () + dataflows: Sequence["DataflowRef"] = () + + def __hash__(self) -> int: + """Returns the organisation's hash.""" + return hash(self.id) class Agency(Organisation, frozen=True, omit_defaults=True): - """Agency class. + """An organisation responsible for maintaining structural metadata. Responsible agency for maintaining artefacts such as statistical classifications, glossaries, @@ -174,6 +181,14 @@ class Agency(Organisation, frozen=True, omit_defaults=True): """ +class DataProvider(Organisation, frozen=True, omit_defaults=True): + """An organisation that provides data or metadata.""" + + +class DataConsumer(Organisation, frozen=True, omit_defaults=True): + """An organisation that collects data or metadata.""" + + class MaintainableArtefact( VersionableArtefact, frozen=True, omit_defaults=True ): @@ -210,3 +225,15 @@ class ItemScheme(MaintainableArtefact, frozen=True, omit_defaults=True): items: Sequence[Item] = () is_partial: bool = False + + +class DataflowRef(MaintainableArtefact, frozen=True, omit_defaults=True): + """Provide core information about a dataflow. + + Attributes: + id: The dataflow identifier (e.g. BIS_MACRO). + agency: The organisation (or unit) responsible for the dataflow. + name: The dataflow name (e.g. MACRO dataflow). + description: Additional descriptive information about the dataflow. + version: The version of the dataflow (e.g. 1.0). + """ From bc346dffae5a4fd2c62c7fd2b9d2b4b384ce31ed Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 13:37:44 +0200 Subject: [PATCH 02/12] Remove file after moving remaining artefacts --- src/pysdmx/model/organisation.py | 89 -------------------------------- 1 file changed, 89 deletions(-) delete mode 100644 src/pysdmx/model/organisation.py diff --git a/src/pysdmx/model/organisation.py b/src/pysdmx/model/organisation.py deleted file mode 100644 index 7797758..0000000 --- a/src/pysdmx/model/organisation.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Model for SDMX Organisations (data providers, agencies, etc). - -Organisations in SDMX can play different roles, such as: - -- Maintenance agencies, i.e. organisations that maintain metadata. -- Data (or metadata) providers, i.e. organisations that provide data or - metadata. -- Data (or metadata) consumers, i.e. organisations that collect data or - metadata. - -Contact details may be included with the information about an organisation. - -Currently, only agencies and providers are supported in pysdmx (as consumers -seem to be underused in SDMX currently). - -Furthermore, in pysdmx, an organisation may reference a list of dataflows -it maintains (if the organisation is an agency) or for which it provides -data (if the organisation is a data provider). -""" - -from typing import Optional, Sequence - -from msgspec import Struct - -from pysdmx.model import Contact - - -class DataflowRef(Struct, frozen=True, omit_defaults=True): - """Provide core information about a dataflow. - - Attributes: - id: The dataflow identifier (e.g. BIS_MACRO). - agency: The organisation (or unit) responsible for the dataflow. - name: The dataflow name (e.g. MACRO dataflow). - description: Additional descriptive information about the dataflow. - version: The version of the dataflow (e.g. 1.0). - """ - - id: str - agency: str - name: Optional[str] = None - description: Optional[str] = None - version: str = "1.0" - - def __str__(self) -> str: - """Returns a human-friendly description.""" - out = self.id - if self.name: - out = f"{out} ({self.name})" - return out - - -class Organisation(Struct, frozen=True, omit_defaults=True): - """An organisation such as a provider of data or a metadata maintainer. - - Central Banks, International Organisations, statistical offices, market - data vendors are typical examples of organisations participating in - statistical data exchanges. - - Organisations may have one or more contact details. - - Attributes: - id: The identifier of the organisation (e.g. BIS). - name: The name of the organisation (e.g. Bank for - International Settlements). - description: Additional descriptive information about - the organisation. - contacts: Contact details (email address, support forum, etc). - dataflows: The dataflows maintained by the organisation if it is - a maintenance agency or for which the organisation provides data - if it is a data provider. - """ - - id: str - name: Optional[str] = None - description: Optional[str] = None - contacts: Sequence[Contact] = () - dataflows: Sequence[DataflowRef] = () - - def __str__(self) -> str: - """Returns a human-friendly description.""" - out = self.id - if self.name: - out = f"{out} ({self.name})" - return out - - def __hash__(self) -> int: - """Returns the organisation's hash.""" - return hash(self.id) From 44ea880b179dbaf9e5f5d85ff59b0ba03be3fab2 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 13:38:52 +0200 Subject: [PATCH 03/12] Update documentation --- src/pysdmx/model/__base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pysdmx/model/__base.py b/src/pysdmx/model/__base.py index c02a570..4868afb 100644 --- a/src/pysdmx/model/__base.py +++ b/src/pysdmx/model/__base.py @@ -172,10 +172,9 @@ def __hash__(self) -> int: class Agency(Organisation, frozen=True, omit_defaults=True): - """An organisation responsible for maintaining structural metadata. + """An organisation that maintains structural metadata. - Responsible agency for maintaining artefacts - such as statistical classifications, glossaries, + This includes statistical classifications, glossaries, structural metadata such as Data and Metadata Structure Definitions, Concepts and Code lists. """ From 7024c8cdaf4e5dff37e3ec7fca1a4d39ccba5375 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 14:01:07 +0200 Subject: [PATCH 04/12] Remove abstract classes from public model API --- src/pysdmx/model/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/pysdmx/model/__init__.py b/src/pysdmx/model/__init__.py index b20aa44..b4f5811 100644 --- a/src/pysdmx/model/__init__.py +++ b/src/pysdmx/model/__init__.py @@ -7,7 +7,14 @@ from re import Pattern from typing import Any -from pysdmx.model.__base import Contact, Item, ItemScheme +from pysdmx.model.__base import ( + Agency, + Contact, + DataflowRef, + DataConsumer, + DataProvider, + Organisation, +) from pysdmx.model.category import Category, CategoryScheme from pysdmx.model.code import ( Code, @@ -38,7 +45,6 @@ ValueMap, ) from pysdmx.model.metadata import MetadataAttribute, MetadataReport -from pysdmx.model.organisation import DataflowRef, Organisation def encoders(obj: Any) -> Any: @@ -78,6 +84,7 @@ def encoders(obj: Any) -> Any: __all__ = [ + "Agency", "ArrayBoundaries", "Category", "CategoryScheme", @@ -89,17 +96,17 @@ def encoders(obj: Any) -> Any: "Concept", "ConceptScheme", "Contact", + "DataConsumer", "DataflowInfo", "DataflowRef", "DataType", "DatePatternMap", + "DataProvider", "Facets", "HierarchicalCode", "Hierarchy", "HierarchyAssociation", "ImplicitComponentMap", - "Item", - "ItemScheme", "StructureMap", "MetadataAttribute", "MetadataReport", From 974a7589c9c4a8717cc734463ee3c5f3a08b390f Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 14:01:18 +0200 Subject: [PATCH 05/12] Remove abstract classes from public model API --- src/pysdmx/model/category.py | 4 ++-- src/pysdmx/model/code.py | 2 +- src/pysdmx/model/concept.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pysdmx/model/category.py b/src/pysdmx/model/category.py index 0e429fd..fe238de 100644 --- a/src/pysdmx/model/category.py +++ b/src/pysdmx/model/category.py @@ -6,8 +6,8 @@ from typing import Iterator, List, Optional, Sequence, Set -from pysdmx.model import Item, ItemScheme -from pysdmx.model.organisation import DataflowRef +from pysdmx.model.__base import Item, ItemScheme +from pysdmx.model import DataflowRef class Category(Item, frozen=False, omit_defaults=True): # type: ignore[misc] diff --git a/src/pysdmx/model/code.py b/src/pysdmx/model/code.py index ddd83e6..6fbf3e0 100644 --- a/src/pysdmx/model/code.py +++ b/src/pysdmx/model/code.py @@ -21,7 +21,7 @@ from msgspec import Struct -from pysdmx.model import Item, ItemScheme +from pysdmx.model.__base import Item, ItemScheme class Code(Item, frozen=True, omit_defaults=True): diff --git a/src/pysdmx/model/concept.py b/src/pysdmx/model/concept.py index c4dc8ac..1931bca 100644 --- a/src/pysdmx/model/concept.py +++ b/src/pysdmx/model/concept.py @@ -16,7 +16,7 @@ from msgspec import Struct -from pysdmx.model import Item, ItemScheme +from pysdmx.model.__base import Item, ItemScheme from pysdmx.model.code import Codelist From 22707abcd0def1e947dafcb11e0f65c26dfbdc7b Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 14:01:34 +0200 Subject: [PATCH 06/12] Use concrete organisation types --- src/pysdmx/model/dataflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pysdmx/model/dataflow.py b/src/pysdmx/model/dataflow.py index de45540..1a91f2d 100644 --- a/src/pysdmx/model/dataflow.py +++ b/src/pysdmx/model/dataflow.py @@ -16,7 +16,7 @@ from pysdmx.model.code import Codelist, Hierarchy from pysdmx.model.concept import DataType, Facets -from pysdmx.model.organisation import Organisation +from pysdmx.model import Agency, DataProvider class Role(str, Enum): @@ -260,11 +260,11 @@ class DataflowInfo(Struct, frozen=True, omit_defaults=True): id: str components: Components - agency: Organisation + agency: Agency name: Optional[str] = None description: Optional[str] = None version: str = "1.0" - providers: Sequence[Organisation] = () + providers: Sequence[DataProvider] = () series_count: Optional[int] = None obs_count: Optional[int] = None start_period: Optional[str] = None From 165e05146a0424dff335c89cf59d458ae13549e1 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 14:01:51 +0200 Subject: [PATCH 07/12] Amend tests following changes to model classes --- tests/model/test_category_scheme.py | 2 +- tests/model/test_dataflow_ref.py | 41 +++++++++++++++++++++-------- tests/model/test_organisation.py | 14 +++++++--- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/tests/model/test_category_scheme.py b/tests/model/test_category_scheme.py index 264ac7a..b0781f8 100644 --- a/tests/model/test_category_scheme.py +++ b/tests/model/test_category_scheme.py @@ -3,7 +3,7 @@ import pytest from pysdmx.model.category import Category, CategoryScheme -from pysdmx.model.organisation import DataflowRef +from pysdmx.model import DataflowRef @pytest.fixture() diff --git a/tests/model/test_dataflow_ref.py b/tests/model/test_dataflow_ref.py index f67e708..7328ef1 100644 --- a/tests/model/test_dataflow_ref.py +++ b/tests/model/test_dataflow_ref.py @@ -29,7 +29,7 @@ def version(): def test_basic_instantiation(dsi, agency): - ds = DataflowRef(dsi, agency) + ds = DataflowRef(id=dsi, agency=agency) assert ds.id == dsi assert ds.name is None @@ -39,7 +39,9 @@ def test_basic_instantiation(dsi, agency): def test_full_instantiation(dsi, name, desc, agency, version): - ds = DataflowRef(dsi, agency, name, desc, version) + ds = DataflowRef( + id=dsi, agency=agency, name=name, description=desc, version=version + ) assert ds.id == dsi assert ds.name == name @@ -49,36 +51,53 @@ def test_full_instantiation(dsi, name, desc, agency, version): def test_immutable(dsi, agency): - ds = DataflowRef(dsi, agency) + ds = DataflowRef(id=dsi, agency=agency) with pytest.raises(AttributeError): ds.name = "Not allowed" def test_equal(dsi, name, desc, agency, version): - ds1 = DataflowRef(dsi, agency, name, desc, version) - ds2 = DataflowRef(dsi, agency, name, desc, version) + ds1 = DataflowRef( + id=dsi, agency=agency, name=name, description=desc, version=version + ) + ds2 = DataflowRef( + id=dsi, agency=agency, name=name, description=desc, version=version + ) assert ds1 == ds2 def test_not_equal(dsi, name, desc, agency, version): - ds1 = DataflowRef(dsi, agency, name, desc, version) - ds2 = DataflowRef(dsi, agency, name, desc, f"{version}.42") + ds1 = DataflowRef( + id=dsi, agency=agency, name=name, description=desc, version=version + ) + ds2 = DataflowRef( + id=dsi, + agency=agency, + name=name, + description=desc, + version=f"{version}.42", + ) assert ds1 != ds2 def test_tostr_id(dsi, agency): - d = DataflowRef(dsi, agency) + d = DataflowRef(id=dsi, agency=agency) s = str(d) - assert s == dsi + assert s == f"id={dsi}, version={d.version}, agency={agency}" def test_tostr_name(dsi, name, desc, agency, version): - d = DataflowRef(dsi, agency, name, desc, version) + d = DataflowRef( + id=dsi, agency=agency, name=name, description=desc, version=version + ) s = str(d) - assert s == f"{dsi} ({name})" + assert s == ( + f"id={dsi}, description={desc}, name={name}, " + f"version={version}, agency={agency}" + ) diff --git a/tests/model/test_organisation.py b/tests/model/test_organisation.py index 25139e4..f62d313 100644 --- a/tests/model/test_organisation.py +++ b/tests/model/test_organisation.py @@ -43,7 +43,13 @@ def test_defaults(id): def test_full_instantiation(id, name, desc, contact, dataflows): - org = Organisation(id, name, desc, contact, dataflows) + org = Organisation( + id=id, + name=name, + description=desc, + contacts=contact, + dataflows=dataflows, + ) assert org.id == id assert org.name == name @@ -77,15 +83,15 @@ def test_tostr_id(id): s = str(o) - assert s == id + assert s == f"id={id}" def test_tostr_name(id, name): - o = Organisation(id, name) + o = Organisation(id=id, name=name) s = str(o) - assert s == f"{id} ({name})" + assert s == f"id={id}, name={name}" def test_equal_has_same_hash(id): From 935301d05f32c29309d47bcf3a64559a3c76897e Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 14:08:22 +0200 Subject: [PATCH 08/12] Add parameter names to organisations --- src/pysdmx/fmr/fusion/org.py | 14 +++++++++++--- src/pysdmx/fmr/sdmx/org.py | 10 ++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/pysdmx/fmr/fusion/org.py b/src/pysdmx/fmr/fusion/org.py index 72a6a41..f4b1bc5 100644 --- a/src/pysdmx/fmr/fusion/org.py +++ b/src/pysdmx/fmr/fusion/org.py @@ -54,9 +54,13 @@ def to_model(self, owner: Optional[str] = None) -> Organisation: c = [c.to_model() for c in self.contacts] oid = f"{owner}.{self.id}" if owner and owner != "SDMX" else self.id if c: - return Organisation(oid, self.names[0].value, d, c) + return Organisation( + id=oid, name=self.names[0].value, description=d, contacts=c + ) else: - return Organisation(oid, self.names[0].value, d) + return Organisation( + id=oid, name=self.names[0].value, description=d + ) class FusionAgencyScheme(Struct, frozen=True): @@ -109,7 +113,11 @@ def to_model( prvs = [o.to_model() for o in self.items] return [ Organisation( - p.id, p.name, p.description, p.contacts, list(paprs[p.id]) + id=p.id, + name=p.name, + description=p.description, + contacts=p.contacts, + dataflows=list(paprs[p.id]), ) for p in prvs ] diff --git a/src/pysdmx/fmr/sdmx/org.py b/src/pysdmx/fmr/sdmx/org.py index 7264407..bdd0f87 100644 --- a/src/pysdmx/fmr/sdmx/org.py +++ b/src/pysdmx/fmr/sdmx/org.py @@ -31,7 +31,11 @@ def to_model( paprs[pr].add(df) return [ Organisation( - p.id, p.name, p.description, p.contacts, list(paprs[p.id]) + id=p.id, + name=p.name, + description=p.description, + contacts=p.contacts, + dataflows=list(paprs[p.id]), ) for p in self.dataProviders ] @@ -80,7 +84,9 @@ class JsonAgencyMessage(Struct, frozen=True): def __add_owner(self, owner: str, a: Organisation) -> Organisation: oid = f"{owner}.{a.id}" if owner != "SDMX" else a.id - return Organisation(oid, a.name, a.description, a.contacts) + return Organisation( + id=oid, name=a.name, description=a.description, contacts=a.contacts + ) def to_model(self) -> Sequence[Organisation]: """Returns the requested list of agencies.""" From 0225e3228d97274a2a4793ac268af6d03f72aac0 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 14:13:27 +0200 Subject: [PATCH 09/12] Address flake8 issues --- src/pysdmx/fmr/__init__.py | 12 ++++++------ src/pysdmx/model/__init__.py | 2 +- src/pysdmx/model/category.py | 3 +-- src/pysdmx/model/dataflow.py | 2 +- src/pysdmx/model/map.py | 4 +++- tests/model/test_category_scheme.py | 2 +- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/pysdmx/fmr/__init__.py b/src/pysdmx/fmr/__init__.py index 060369e..ec405c6 100644 --- a/src/pysdmx/fmr/__init__.py +++ b/src/pysdmx/fmr/__init__.py @@ -224,9 +224,9 @@ def __fetch(self, url: str, is_ref_meta: bool = False) -> bytes: try: if is_ref_meta and self.format == Format.SDMX_JSON: h = self.headers.copy() - h["Accept"] = ( - "application/vnd.sdmx.metadata+json;version=2.0.0" - ) + h[ + "Accept" + ] = "application/vnd.sdmx.metadata+json;version=2.0.0" else: h = self.headers r = client.get(url, headers=h) @@ -556,9 +556,9 @@ async def __fetch(self, url: str, is_ref_meta: bool = False) -> bytes: try: if is_ref_meta and self.format == Format.SDMX_JSON: h = self.headers.copy() - h["Accept"] = ( - "application/vnd.sdmx.metadata+json;version=2.0.0" - ) + h[ + "Accept" + ] = "application/vnd.sdmx.metadata+json;version=2.0.0" else: h = self.headers r = await client.get(url, headers=h) diff --git a/src/pysdmx/model/__init__.py b/src/pysdmx/model/__init__.py index b4f5811..b2b1338 100644 --- a/src/pysdmx/model/__init__.py +++ b/src/pysdmx/model/__init__.py @@ -10,8 +10,8 @@ from pysdmx.model.__base import ( Agency, Contact, - DataflowRef, DataConsumer, + DataflowRef, DataProvider, Organisation, ) diff --git a/src/pysdmx/model/category.py b/src/pysdmx/model/category.py index fe238de..75bfb9f 100644 --- a/src/pysdmx/model/category.py +++ b/src/pysdmx/model/category.py @@ -6,8 +6,7 @@ from typing import Iterator, List, Optional, Sequence, Set -from pysdmx.model.__base import Item, ItemScheme -from pysdmx.model import DataflowRef +from pysdmx.model.__base import DataflowRef, Item, ItemScheme class Category(Item, frozen=False, omit_defaults=True): # type: ignore[misc] diff --git a/src/pysdmx/model/dataflow.py b/src/pysdmx/model/dataflow.py index 1a91f2d..b019a54 100644 --- a/src/pysdmx/model/dataflow.py +++ b/src/pysdmx/model/dataflow.py @@ -14,9 +14,9 @@ from msgspec import Struct +from pysdmx.model.__base import Agency, DataProvider from pysdmx.model.code import Codelist, Hierarchy from pysdmx.model.concept import DataType, Facets -from pysdmx.model import Agency, DataProvider class Role(str, Enum): diff --git a/src/pysdmx/model/map.py b/src/pysdmx/model/map.py index 8ffe867..7c9f564 100644 --- a/src/pysdmx/model/map.py +++ b/src/pysdmx/model/map.py @@ -421,7 +421,9 @@ def __len__(self) -> int: """Return the number of mapping rules in the structure map.""" return len(self.maps) - def __getitem__(self, id_: str) -> Optional[ + def __getitem__( + self, id_: str + ) -> Optional[ Sequence[ Union[ ComponentMap, diff --git a/tests/model/test_category_scheme.py b/tests/model/test_category_scheme.py index b0781f8..6af1a73 100644 --- a/tests/model/test_category_scheme.py +++ b/tests/model/test_category_scheme.py @@ -2,8 +2,8 @@ import pytest +from pysdmx.model.__base import DataflowRef from pysdmx.model.category import Category, CategoryScheme -from pysdmx.model import DataflowRef @pytest.fixture() From 374507ead6c95fcb20a29a50ec645deec4b57b0f Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 14:42:03 +0200 Subject: [PATCH 10/12] Use more concrete types --- src/pysdmx/fmr/__init__.py | 11 ++++---- src/pysdmx/fmr/fusion/dataflow.py | 12 ++++++--- src/pysdmx/fmr/fusion/org.py | 45 ++++++++++++++++++++++--------- src/pysdmx/fmr/sdmx/dataflow.py | 12 ++++++--- src/pysdmx/fmr/sdmx/org.py | 20 +++++++------- 5 files changed, 67 insertions(+), 33 deletions(-) diff --git a/src/pysdmx/fmr/__init__.py b/src/pysdmx/fmr/__init__.py index ec405c6..c9d17ec 100644 --- a/src/pysdmx/fmr/__init__.py +++ b/src/pysdmx/fmr/__init__.py @@ -19,15 +19,16 @@ from pysdmx.fmr.reader import Deserializer from pysdmx.fmr.sdmx import deserializers as sdmx_deserializers from pysdmx.model import ( + Agency, CategoryScheme, Codelist, ConceptScheme, DataflowInfo, + DataProvider, Hierarchy, HierarchyAssociation, MetadataReport, MultiRepresentationMap, - Organisation, RepresentationMap, Schema, StructureMap, @@ -247,7 +248,7 @@ def __get_hierarchies_for_pra( out = self.__fetch(super()._url("ha_pra", agency, pra, version)) return super()._out(out, self.deser.hier_assoc) - def get_agencies(self, agency: str) -> Sequence[Organisation]: + def get_agencies(self, agency: str) -> Sequence[Agency]: """Get the list of **sub-agencies** for the supplied agency. Args: @@ -264,7 +265,7 @@ def get_providers( self, agency: str, with_flows: bool = False, - ) -> Sequence[Organisation]: + ) -> Sequence[DataProvider]: """Get the list of **data providers** for the supplied agency. Args: @@ -579,7 +580,7 @@ async def __get_hierarchies_for_pra( out = await self.__fetch(super()._url("ha_pra", agency, pra, version)) return super()._out(out, self.deser.hier_assoc) - async def get_agencies(self, agency: str) -> Sequence[Organisation]: + async def get_agencies(self, agency: str) -> Sequence[Agency]: """Get the list of **sub-agencies** for the supplied agency. Args: @@ -594,7 +595,7 @@ async def get_agencies(self, agency: str) -> Sequence[Organisation]: async def get_providers( self, agency: str, with_flows: bool = False - ) -> Sequence[Organisation]: + ) -> Sequence[DataProvider]: """Get the list of **data providers** for the supplied agency. Args: diff --git a/src/pysdmx/fmr/fusion/dataflow.py b/src/pysdmx/fmr/fusion/dataflow.py index 4a557a3..0a217b4 100644 --- a/src/pysdmx/fmr/fusion/dataflow.py +++ b/src/pysdmx/fmr/fusion/dataflow.py @@ -6,7 +6,13 @@ from pysdmx.fmr.fusion.core import FusionString from pysdmx.fmr.fusion.org import FusionProviderScheme -from pysdmx.model import Components, DataflowInfo, DataflowRef, Organisation +from pysdmx.model import ( + Agency, + Components, + DataflowInfo, + DataflowRef, + DataProvider, +) class FusionDataflowRef(Struct, frozen=True, rename={"agency": "agencyId"}): @@ -64,7 +70,7 @@ def to_model( self, components: Components, agency: str, id_: str, version: str ) -> DataflowInfo: """Returns the requested dataflow details.""" - prvs: List[Organisation] = [] + prvs: List[DataProvider] = [] for dps in self.DataProviderScheme: prvs.extend(dps.to_model([])) df = list( @@ -76,7 +82,7 @@ def to_model( return DataflowInfo( df.id, components, - Organisation(df.agency), + Agency(df.agency), df.names[0].value, df.descriptions[0].value if df.descriptions else None, df.version, diff --git a/src/pysdmx/fmr/fusion/org.py b/src/pysdmx/fmr/fusion/org.py index f4b1bc5..79998e3 100644 --- a/src/pysdmx/fmr/fusion/org.py +++ b/src/pysdmx/fmr/fusion/org.py @@ -6,7 +6,7 @@ from msgspec import Struct from pysdmx.fmr.fusion.core import FusionString -from pysdmx.model import Contact, DataflowRef, Organisation +from pysdmx.model import Agency, Contact, DataflowRef, DataProvider from pysdmx.util import parse_urn @@ -40,7 +40,7 @@ def to_model(self) -> Contact: ) -class FusionOrg(Struct, frozen=True): +class FusionAgency(Struct, frozen=True): """Fusion-JSON payload for an organisation.""" id: str @@ -48,17 +48,38 @@ class FusionOrg(Struct, frozen=True): descriptions: Optional[Sequence[FusionString]] = None contacts: Sequence[FusionContact] = () - def to_model(self, owner: Optional[str] = None) -> Organisation: + def to_model(self, owner: Optional[str] = None) -> Agency: """Converts a FusionOrg to a standard Organisation.""" d = self.descriptions[0].value if self.descriptions else None c = [c.to_model() for c in self.contacts] oid = f"{owner}.{self.id}" if owner and owner != "SDMX" else self.id if c: - return Organisation( + return Agency( id=oid, name=self.names[0].value, description=d, contacts=c ) else: - return Organisation( + return Agency(id=oid, name=self.names[0].value, description=d) + + +class FusionProvider(Struct, frozen=True): + """Fusion-JSON payload for an organisation.""" + + id: str + names: Sequence[FusionString] + descriptions: Optional[Sequence[FusionString]] = None + contacts: Sequence[FusionContact] = () + + def to_model(self, owner: Optional[str] = None) -> DataProvider: + """Converts a FusionOrg to a standard Organisation.""" + d = self.descriptions[0].value if self.descriptions else None + c = [c.to_model() for c in self.contacts] + oid = f"{owner}.{self.id}" if owner and owner != "SDMX" else self.id + if c: + return DataProvider( + id=oid, name=self.names[0].value, description=d, contacts=c + ) + else: + return DataProvider( id=oid, name=self.names[0].value, description=d ) @@ -67,9 +88,9 @@ class FusionAgencyScheme(Struct, frozen=True): """Fusion-JSON payload for an agency scheme.""" agencyId: str - items: Sequence[FusionOrg] + items: Sequence[FusionAgency] - def to_model(self) -> Sequence[Organisation]: + def to_model(self) -> Sequence[Agency]: """Converts a FusionAgencyScheme to a list of Organisations.""" return [o.to_model(self.agencyId) for o in self.items] @@ -79,7 +100,7 @@ class FusionAgencyMessage(Struct, frozen=True): AgencyScheme: Sequence[FusionAgencyScheme] - def to_model(self) -> Sequence[Organisation]: + def to_model(self) -> Sequence[Agency]: """Returns the requested list of agencies.""" return self.AgencyScheme[0].to_model() @@ -94,7 +115,7 @@ class FusionProvisionAgreement(Struct, frozen=True): class FusionProviderScheme(Struct, frozen=True): """Fusion-JSON payload for a data provider scheme.""" - items: Sequence[FusionOrg] + items: Sequence[FusionProvider] def __get_df_ref(self, ref: str) -> DataflowRef: a = parse_urn(ref) @@ -102,7 +123,7 @@ def __get_df_ref(self, ref: str) -> DataflowRef: def to_model( self, pas: Sequence[FusionProvisionAgreement] - ) -> Sequence[Organisation]: + ) -> Sequence[DataProvider]: """Converts a FusionProviderScheme to a list of Organisations.""" if pas: paprs: Dict[str, Set[DataflowRef]] = defaultdict(set) @@ -112,7 +133,7 @@ def to_model( paprs[pr].add(df) prvs = [o.to_model() for o in self.items] return [ - Organisation( + DataProvider( id=p.id, name=p.name, description=p.description, @@ -131,6 +152,6 @@ class FusionProviderMessage(Struct, frozen=True): DataProviderScheme: Sequence[FusionProviderScheme] ProvisionAgreement: Sequence[FusionProvisionAgreement] = () - def to_model(self) -> Sequence[Organisation]: + def to_model(self) -> Sequence[DataProvider]: """Returns the requested list of providers.""" return self.DataProviderScheme[0].to_model(self.ProvisionAgreement) diff --git a/src/pysdmx/fmr/sdmx/dataflow.py b/src/pysdmx/fmr/sdmx/dataflow.py index 2db98ad..61ea1ca 100644 --- a/src/pysdmx/fmr/sdmx/dataflow.py +++ b/src/pysdmx/fmr/sdmx/dataflow.py @@ -5,7 +5,13 @@ from msgspec import Struct from pysdmx.fmr.sdmx.org import JsonDataProviderScheme -from pysdmx.model import Components, DataflowInfo, DataflowRef, Organisation +from pysdmx.model import ( + Agency, + Components, + DataflowInfo, + DataflowRef, + DataProvider, +) class JsonDataflowRef(Struct, frozen=True, rename={"agency": "agencyID"}): @@ -59,7 +65,7 @@ def to_model( self, components: Components, agency: str, id_: str, version: str ) -> DataflowInfo: """Returns the requested dataflow details.""" - prvs: List[Organisation] = [] + prvs: List[DataProvider] = [] for dps in self.dataProviderSchemes: prvs.extend(dps.dataProviders) df = list( @@ -71,7 +77,7 @@ def to_model( return DataflowInfo( df.id, components, - Organisation(df.agency), + Agency(df.agency), df.name, df.description, df.version, diff --git a/src/pysdmx/fmr/sdmx/org.py b/src/pysdmx/fmr/sdmx/org.py index bdd0f87..1039a66 100644 --- a/src/pysdmx/fmr/sdmx/org.py +++ b/src/pysdmx/fmr/sdmx/org.py @@ -6,14 +6,14 @@ from msgspec import Struct from pysdmx.fmr.sdmx.pa import JsonProvisionAgreement -from pysdmx.model import DataflowRef, Organisation +from pysdmx.model import Agency, DataflowRef, DataProvider from pysdmx.util import parse_urn class JsonDataProviderScheme(Struct, frozen=True): """SDMX-JSON payload for a data provider scheme.""" - dataProviders: Sequence[Organisation] + dataProviders: Sequence[DataProvider] def __get_df_ref(self, ref: str) -> DataflowRef: a = parse_urn(ref) @@ -21,7 +21,7 @@ def __get_df_ref(self, ref: str) -> DataflowRef: def to_model( self, pas: Sequence[JsonProvisionAgreement] - ) -> Sequence[Organisation]: + ) -> Sequence[DataProvider]: """Converts a JsonDataProviderScheme to a list of Organisations.""" if pas: paprs: Dict[str, Set[DataflowRef]] = defaultdict(set) @@ -30,7 +30,7 @@ def to_model( pr = pa.dataProvider[pa.dataProvider.rindex(".") + 1 :] paprs[pr].add(df) return [ - Organisation( + DataProvider( id=p.id, name=p.name, description=p.description, @@ -49,7 +49,7 @@ class JsonDataProviderSchemes(Struct, frozen=True): dataProviderSchemes: Sequence[JsonDataProviderScheme] provisionAgreements: Sequence[JsonProvisionAgreement] = () - def to_model(self) -> Sequence[Organisation]: + def to_model(self) -> Sequence[DataProvider]: """Converts a JsonDataProviderSchemes to a list of Organisations.""" return self.dataProviderSchemes[0].to_model(self.provisionAgreements) @@ -59,7 +59,7 @@ class JsonProviderMessage(Struct, frozen=True): data: JsonDataProviderSchemes - def to_model(self) -> Sequence[Organisation]: + def to_model(self) -> Sequence[DataProvider]: """Returns the requested list of providers.""" return self.data.to_model() @@ -68,7 +68,7 @@ class JsonAgencyScheme(Struct, frozen=True): """SDMX-JSON payload for an agency scheme.""" agencyID: str - agencies: Sequence[Organisation] + agencies: Sequence[Agency] class JsonAgencySchemes(Struct, frozen=True): @@ -82,13 +82,13 @@ class JsonAgencyMessage(Struct, frozen=True): data: JsonAgencySchemes - def __add_owner(self, owner: str, a: Organisation) -> Organisation: + def __add_owner(self, owner: str, a: Agency) -> Agency: oid = f"{owner}.{a.id}" if owner != "SDMX" else a.id - return Organisation( + return Agency( id=oid, name=a.name, description=a.description, contacts=a.contacts ) - def to_model(self) -> Sequence[Organisation]: + def to_model(self) -> Sequence[Agency]: """Returns the requested list of agencies.""" return [ self.__add_owner(self.data.agencySchemes[0].agencyID, a) From f46606b39d45dda132bcd21326d69431048ad784 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Fri, 3 May 2024 16:23:25 +0200 Subject: [PATCH 11/12] Apply black formatting --- src/pysdmx/fmr/__init__.py | 12 ++++++------ src/pysdmx/model/map.py | 4 +--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/pysdmx/fmr/__init__.py b/src/pysdmx/fmr/__init__.py index c9d17ec..43710ff 100644 --- a/src/pysdmx/fmr/__init__.py +++ b/src/pysdmx/fmr/__init__.py @@ -225,9 +225,9 @@ def __fetch(self, url: str, is_ref_meta: bool = False) -> bytes: try: if is_ref_meta and self.format == Format.SDMX_JSON: h = self.headers.copy() - h[ - "Accept" - ] = "application/vnd.sdmx.metadata+json;version=2.0.0" + h["Accept"] = ( + "application/vnd.sdmx.metadata+json;version=2.0.0" + ) else: h = self.headers r = client.get(url, headers=h) @@ -557,9 +557,9 @@ async def __fetch(self, url: str, is_ref_meta: bool = False) -> bytes: try: if is_ref_meta and self.format == Format.SDMX_JSON: h = self.headers.copy() - h[ - "Accept" - ] = "application/vnd.sdmx.metadata+json;version=2.0.0" + h["Accept"] = ( + "application/vnd.sdmx.metadata+json;version=2.0.0" + ) else: h = self.headers r = await client.get(url, headers=h) diff --git a/src/pysdmx/model/map.py b/src/pysdmx/model/map.py index 7c9f564..8ffe867 100644 --- a/src/pysdmx/model/map.py +++ b/src/pysdmx/model/map.py @@ -421,9 +421,7 @@ def __len__(self) -> int: """Return the number of mapping rules in the structure map.""" return len(self.maps) - def __getitem__( - self, id_: str - ) -> Optional[ + def __getitem__(self, id_: str) -> Optional[ Sequence[ Union[ ComponentMap, From bbca1ea47445932bd49a27082c56fb64afe4ece1 Mon Sep 17 00:00:00 2001 From: Xavier Sosnovsky Date: Mon, 13 May 2024 10:10:22 +0200 Subject: [PATCH 12/12] Add param names to DataflowRef and DataflowInfo --- src/pysdmx/fmr/fusion/dataflow.py | 26 ++++++++++++++------------ src/pysdmx/fmr/fusion/org.py | 2 +- src/pysdmx/fmr/sdmx/dataflow.py | 24 ++++++++++++------------ src/pysdmx/fmr/sdmx/org.py | 2 +- tests/model/test_category.py | 2 +- tests/model/test_organisation.py | 4 ++-- 6 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/pysdmx/fmr/fusion/dataflow.py b/src/pysdmx/fmr/fusion/dataflow.py index 0a217b4..e75f5a4 100644 --- a/src/pysdmx/fmr/fusion/dataflow.py +++ b/src/pysdmx/fmr/fusion/dataflow.py @@ -27,11 +27,13 @@ class FusionDataflowRef(Struct, frozen=True, rename={"agency": "agencyId"}): def to_model(self) -> DataflowRef: """Converts a FusionDataflowRef to a standard dataflow ref.""" return DataflowRef( - self.id, - self.agency, - self.names[0].value if self.names else None, - self.descriptions[0].value if self.descriptions else None, - self.version, + id=self.id, + agency=self.agency, + name=self.names[0].value if self.names else None, + description=( + self.descriptions[0].value if self.descriptions else None + ), + version=self.version, ) @@ -80,12 +82,12 @@ def to_model( ) )[0] return DataflowInfo( - df.id, - components, - Agency(df.agency), - df.names[0].value, - df.descriptions[0].value if df.descriptions else None, - df.version, - prvs, + id=df.id, + components=components, + agency=Agency(df.agency), + name=df.names[0].value, + description=df.descriptions[0].value if df.descriptions else None, + version=df.version, + providers=prvs, dsd_ref=df.dataStructureRef, ) diff --git a/src/pysdmx/fmr/fusion/org.py b/src/pysdmx/fmr/fusion/org.py index 79998e3..6c3e9af 100644 --- a/src/pysdmx/fmr/fusion/org.py +++ b/src/pysdmx/fmr/fusion/org.py @@ -119,7 +119,7 @@ class FusionProviderScheme(Struct, frozen=True): def __get_df_ref(self, ref: str) -> DataflowRef: a = parse_urn(ref) - return DataflowRef(a.id, a.agency, version=a.version) + return DataflowRef(id=a.id, agency=a.agency, version=a.version) def to_model( self, pas: Sequence[FusionProvisionAgreement] diff --git a/src/pysdmx/fmr/sdmx/dataflow.py b/src/pysdmx/fmr/sdmx/dataflow.py index 61ea1ca..754e0db 100644 --- a/src/pysdmx/fmr/sdmx/dataflow.py +++ b/src/pysdmx/fmr/sdmx/dataflow.py @@ -26,11 +26,11 @@ class JsonDataflowRef(Struct, frozen=True, rename={"agency": "agencyID"}): def to_model(self) -> DataflowRef: """Converts a JsonDataflowRef to a standard dataflow ref.""" return DataflowRef( - self.id, - self.agency, - self.name, - self.description, - self.version, + id=self.id, + agency=self.agency, + name=self.name, + description=self.description, + version=self.version, ) @@ -75,13 +75,13 @@ def to_model( ) )[0] return DataflowInfo( - df.id, - components, - Agency(df.agency), - df.name, - df.description, - df.version, - prvs, + id=df.id, + components=components, + agency=Agency(df.agency), + name=df.name, + description=df.description, + version=df.version, + providers=prvs, dsd_ref=df.structure, ) diff --git a/src/pysdmx/fmr/sdmx/org.py b/src/pysdmx/fmr/sdmx/org.py index 1039a66..2356ed8 100644 --- a/src/pysdmx/fmr/sdmx/org.py +++ b/src/pysdmx/fmr/sdmx/org.py @@ -17,7 +17,7 @@ class JsonDataProviderScheme(Struct, frozen=True): def __get_df_ref(self, ref: str) -> DataflowRef: a = parse_urn(ref) - return DataflowRef(a.id, a.agency, version=a.version) + return DataflowRef(id=a.id, agency=a.agency, version=a.version) def to_model( self, pas: Sequence[JsonProvisionAgreement] diff --git a/tests/model/test_category.py b/tests/model/test_category.py index 269d0a4..7e4e624 100644 --- a/tests/model/test_category.py +++ b/tests/model/test_category.py @@ -34,7 +34,7 @@ def test_default(id): def test_full_instantiation(id, name, desc): cats = [Category(id="chld", name="Child")] - flows = [DataflowRef("EXR", "BIS")] + flows = [DataflowRef(id="EXR", agency="BIS")] c = Category( id=id, name=name, description=desc, categories=cats, dataflows=flows diff --git a/tests/model/test_organisation.py b/tests/model/test_organisation.py index f62d313..50d1ccf 100644 --- a/tests/model/test_organisation.py +++ b/tests/model/test_organisation.py @@ -25,8 +25,8 @@ def contact(): @pytest.fixture() def dataflows(): - df1 = DataflowRef("DF1", "TEST") - df2 = DataflowRef("DF2", "Also TEST") + df1 = DataflowRef(id="DF1", name="TEST", agency="T1") + df2 = DataflowRef(id="DF2", name="Also TEST", agency="T1") return [df1, df2]