diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1cd9443..af72f6e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,17 +17,6 @@ repos: - id: ruff args: [ --fix, --exit-non-zero-on-fix ] - - repo: https://github.com/psf/black - rev: 23.3.0 - hooks: - - id: black - - - repo: https://github.com/timothycrosley/isort - rev: 5.12.0 - hooks: - - id: isort - args: ["--profile", "black"] - - repo: https://github.com/asottile/pyupgrade rev: v3.3.1 hooks: diff --git a/app/core/src/sghi/idr/client/core/domain/interfaces/__init__.py b/app/core/src/sghi/idr/client/core/domain/interfaces/__init__.py index e6c8365..d415468 100644 --- a/app/core/src/sghi/idr/client/core/domain/interfaces/__init__.py +++ b/app/core/src/sghi/idr/client/core/domain/interfaces/__init__.py @@ -1,5 +1,5 @@ from .base import DomainObject, IdentifiableDomainObject, NamedDomainObject -from .etl_protocol import ETLProtocol +from .etl_protocol import ETLProtocol, ETLProtocolSupplier from .metadata import ( DataSinkMetadata, DataSourceMetadata, @@ -35,6 +35,7 @@ "DataSourceMetadata", "DomainObject", "ETLProtocol", + "ETLProtocolSupplier", "ExtractMetadata", "ExtractProcessor", "IdentifiableDomainObject", diff --git a/app/core/src/sghi/idr/client/core/domain/interfaces/etl_protocol.py b/app/core/src/sghi/idr/client/core/domain/interfaces/etl_protocol.py index 109b0ef..3a80d41 100644 --- a/app/core/src/sghi/idr/client/core/domain/interfaces/etl_protocol.py +++ b/app/core/src/sghi/idr/client/core/domain/interfaces/etl_protocol.py @@ -2,8 +2,8 @@ ETL Protocol Definition. """ from abc import ABCMeta, abstractmethod -from collections.abc import Callable -from typing import Generic, TypeVar +from collections.abc import Callable, Iterable +from typing import Any, Generic, TypeVar from .base import IdentifiableDomainObject, NamedDomainObject from .operations import ( @@ -83,3 +83,11 @@ def metadata_supplier(self) -> MetadataSupplier[_DS, _DM, _EM]: @abstractmethod def upload_metadata_factory(self) -> UploadMetadataFactory[_UM, _EM]: ... + + +class ETLProtocolSupplier(metaclass=ABCMeta): + @abstractmethod + def get_protocols( + self, + ) -> Iterable[ETLProtocol[Any, Any, Any, Any, Any, Any]]: + ... diff --git a/app/core/src/sghi/idr/client/core/lib/module_loading.py b/app/core/src/sghi/idr/client/core/lib/module_loading.py index 6ed3d78..1c4ea8f 100644 --- a/app/core/src/sghi/idr/client/core/lib/module_loading.py +++ b/app/core/src/sghi/idr/client/core/lib/module_loading.py @@ -1,9 +1,7 @@ -# The contents of this module are copied from Django sources. import inspect -import sys -from importlib import import_module -from types import ModuleType -from typing import TypeVar, cast +from typing import Any, Final, TypeVar, cast + +from importlib_metadata import EntryPoint # ============================================================================= # TYPES @@ -13,20 +11,10 @@ # ============================================================================= -# HELPERS +# CONSTANTS # ============================================================================= - -def _cached_import(module_path: str, class_name: str) -> ModuleType: - modules = sys.modules - if module_path not in modules or ( - # Module is not fully initialized. - getattr(modules[module_path], "__spec__", None) is not None - and getattr(modules[module_path].__spec__, "_initializing", False) - is True - ): # pragma: no branch - import_module(module_path) - return getattr(modules[module_path], class_name) +_UNKNOWN_STR: Final[str] = "UNKNOWN" # ============================================================================= @@ -34,33 +22,32 @@ def _cached_import(module_path: str, class_name: str) -> ModuleType: # ============================================================================= -def import_string(dotted_path: str) -> ModuleType: +def import_string(dotted_path: str) -> Any: # noqa: ANN401 """ Import a dotted module path and return the attribute/class designated by the last name in the path. Raise ``ImportError`` if the import failed. + The `dotted_path` should conform to the format defined by the Python + packaging conventions. See `the packaging docs on entry points + `_ + for more information. + :param dotted_path: A dotted path to an attribute or class. :return: The attribute/class designated by the last name in the path. :raise ImportError: If the import fails for some reason. """ + entry_point = EntryPoint( + name=_UNKNOWN_STR, + group=_UNKNOWN_STR, + value=dotted_path, + ) try: - module_path, class_name = dotted_path.rsplit(".", 1) - except ValueError as err: - err_msg: str = f'"{dotted_path}" does not look like a module path.' - raise ImportError(err_msg, path=dotted_path) from err - - try: - return _cached_import(module_path, class_name) - except AttributeError as err: - err_msg: str = ( - 'Module "{}" does not define a "{}" attribute/class'.format( - module_path, - class_name, - ) - ) - raise ImportError(err_msg, name=class_name, path=module_path) from err + return entry_point.load() + except AttributeError as exp: + _err_msg: str = str(exp) + raise ImportError(_err_msg) from exp def import_string_as_klass( diff --git a/app/mods/common/pyproject.toml b/app/mods/common/pyproject.toml index b0a6dff..8cbc42e 100644 --- a/app/mods/common/pyproject.toml +++ b/app/mods/common/pyproject.toml @@ -39,6 +39,10 @@ name = "sghi-idr-client-common" readme = "README.md" requires-python = ">=3.10" # Support Python 3.10+. +[project.entry-points."sghi.idr.client.etl_protocol"] +from_definitions = "sghi.idr.client.common.domain:FromDefinitionsETLProtocolSupplier" +from_factories = "sghi.idr.client.common.domain:FromFactoriesETLProtocolSupplier" + [project.urls] changelog = "https://github.com/savannahghi/idr-client/blob/develop/docs/CHANGELOG.md" documentation = "https://github.com/savannahghi/idr-client/blob/develop/README.md" diff --git a/app/mods/common/src/sghi/idr/client/common/domain/__init__.py b/app/mods/common/src/sghi/idr/client/common/domain/__init__.py index 22b0d32..0877ee5 100644 --- a/app/mods/common/src/sghi/idr/client/common/domain/__init__.py +++ b/app/mods/common/src/sghi/idr/client/common/domain/__init__.py @@ -1,5 +1,11 @@ -from .etl_protocol import SimpleETLProtocol +from .etl_protocol import ( + FromDefinitionsETLProtocolSupplier, + FromFactoriesETLProtocolSupplier, + SimpleETLProtocol, +) __all__ = [ + "FromDefinitionsETLProtocolSupplier", + "FromFactoriesETLProtocolSupplier", "SimpleETLProtocol", ] diff --git a/app/mods/common/src/sghi/idr/client/common/domain/etl_protocol.py b/app/mods/common/src/sghi/idr/client/common/domain/etl_protocol.py index 79a2d24..0021660 100644 --- a/app/mods/common/src/sghi/idr/client/common/domain/etl_protocol.py +++ b/app/mods/common/src/sghi/idr/client/common/domain/etl_protocol.py @@ -1,5 +1,5 @@ -from collections.abc import Callable -from typing import Generic, TypeVar +from collections.abc import Callable, Iterable +from typing import Any, Generic, TypeVar, assert_never from attrs import define, field from sghi.idr.client.core.domain import ( @@ -11,6 +11,7 @@ DataSource, DataSourceMetadata, ETLProtocol, + ETLProtocolSupplier, ExtractMetadata, ExtractProcessor, MetadataConsumer, @@ -19,6 +20,14 @@ UploadMetadata, UploadMetadataFactory, ) +from sghi.idr.client.core.lib import ImproperlyConfiguredError, type_fqn + +from ..lib import ( + ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY, + ETL_PROTOCOL_FACTORIES_CONFIG_KEY, + ETLProtocolFactory, + ProtocolDefinition, +) # ============================================================================= # TYPES @@ -31,9 +40,15 @@ _RD = TypeVar("_RD", bound=RawData) _UM = TypeVar("_UM", bound=UploadMetadata) +_EP = ETLProtocol[Any, Any, Any, Any, Any, Any] + # ============================================================================= -# CONCRETE ETL PROTOCOL DEFINITION +# HELPERS +# ============================================================================= + +# ============================================================================= +# CONCRETE ETL PROTOCOL DEFINITIONS # ============================================================================= @@ -57,11 +72,13 @@ class SimpleETLProtocol( _upload_metadata_factory: UploadMetadataFactory[_UM, _EM] = field() @property - def data_sink_factory(self) -> Callable[[_DS], DataSink]: + def data_sink_factory(self) -> Callable[[_DS], DataSink[_DS, _UM, _CD]]: return self._data_sink_factory @property - def data_source_factory(self) -> Callable[[_DM], DataSource]: + def data_source_factory( + self, + ) -> Callable[[_DM], DataSource[_DM, _EM, _RD]]: return self._data_source_factory @property @@ -81,3 +98,109 @@ def metadata_supplier(self) -> MetadataSupplier[_DS, _DM, _EM]: @property def upload_metadata_factory(self) -> UploadMetadataFactory[_UM, _EM]: return self._upload_metadata_factory + + +# ============================================================================= +# ETL PROTOCOL SUPPLIERS +# ============================================================================= + + +class FromDefinitionsETLProtocolSupplier(ETLProtocolSupplier): + """ + Load :class:`ETLProtocol` instances from ETLProtocol definitions on the + config. + """ + + def get_protocols(self) -> Iterable[_EP]: + from sghi.idr.client.core import settings + + proto_definitions: Iterable[ProtocolDefinition] + proto_definitions = settings.get( + setting=ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY, + default=(), + ) + return map( + self._proto_definition_to_proto_instance, + proto_definitions, + ) + + @classmethod + def _proto_definition_to_proto_instance( + cls, + protocol_definition: ProtocolDefinition, + ) -> _EP: + upf_def = protocol_definition["upload_metadata_factory"] + upf: UploadMetadataFactory = cls._get_upload_meta_factory_instance( + upf_def, + ) + return SimpleETLProtocol( + id=protocol_definition["id"], + name=protocol_definition["name"], + description=protocol_definition.get("description"), + data_sink_factory=protocol_definition["data_sink_factory"], + data_source_factory=protocol_definition["data_source_factory"], + extract_processor_factory=protocol_definition[ + "extract_processor_factory" + ], + metadata_consumer=protocol_definition[ + "metadata_consumer_factory" + ](), + metadata_supplier=protocol_definition[ + "metadata_supplier_factory" + ](), + upload_metadata_factory=upf, + ) + + @staticmethod + def _get_upload_meta_factory_instance( + upload_meta_factory: UploadMetadataFactory + | Callable[[], UploadMetadataFactory], + ) -> UploadMetadataFactory: + match upload_meta_factory: + case UploadMetadataFactory(): + return upload_meta_factory + case Callable(): + return upload_meta_factory() + case _: + assert_never(upload_meta_factory) + + +class FromFactoriesETLProtocolSupplier(ETLProtocolSupplier): + """ + Load :class:`ETLProtocol` instances from ETLProtocol factories on the + config. + """ + + def get_protocols(self) -> Iterable[_EP]: + from sghi.idr.client.core import settings + + proto_factories: Iterable[ETLProtocolFactory] + proto_factories = settings.get( + setting=ETL_PROTOCOL_FACTORIES_CONFIG_KEY, + default=(), + ) + + return map(self._proto_factory_to_instance, proto_factories) + + @staticmethod + def _proto_factory_to_instance(proto_factory: ETLProtocolFactory) -> _EP: + try: + _etl_proto_instance: _EP = proto_factory() + except Exception as exp: # noqa: BLE001 + _err_msg: str = ( + "Unable to create an ETLProtocol instance from factory. The " + "cause was: '{}'".format(str(exp)) + ) + raise RuntimeError(_err_msg) from exp + + if not isinstance(_etl_proto_instance, ETLProtocol): + _err_msg: str = ( + "Invalid ETLProtocol, the factory '{}' returned an instance " + "that is not a subclass of " + "'app.core.domain.ETLProtocol'.".format( + type_fqn(proto_factory), + ) + ) + raise ImproperlyConfiguredError(message=_err_msg) + + return _etl_proto_instance diff --git a/app/mods/common/src/sghi/idr/client/common/lib/__init__.py b/app/mods/common/src/sghi/idr/client/common/lib/__init__.py index 935950b..7e741f8 100644 --- a/app/mods/common/src/sghi/idr/client/common/lib/__init__.py +++ b/app/mods/common/src/sghi/idr/client/common/lib/__init__.py @@ -1,6 +1,5 @@ -from .config import LocationIDInitializer, LocationNameInitializer +from .config import * # noqa: F403 +from .config import __all__ as _all_config -__all__ = [ - "LocationIDInitializer", - "LocationNameInitializer", -] +__all__ = [] +__all__ += _all_config # type: ignore diff --git a/app/mods/common/src/sghi/idr/client/common/lib/config/__init__.py b/app/mods/common/src/sghi/idr/client/common/lib/config/__init__.py new file mode 100644 index 0000000..04375d6 --- /dev/null +++ b/app/mods/common/src/sghi/idr/client/common/lib/config/__init__.py @@ -0,0 +1,20 @@ +from .etl_protocols_config import ( + ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY, + ETL_PROTOCOL_FACTORIES_CONFIG_KEY, + ETLProtocolDefinitionsInitializer, + ETLProtocolFactoriesInitializer, + ETLProtocolFactory, + ProtocolDefinition, +) +from .location_config import LocationIDInitializer, LocationNameInitializer + +__all__ = [ + "ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY", + "ETL_PROTOCOL_FACTORIES_CONFIG_KEY", + "ETLProtocolDefinitionsInitializer", + "ETLProtocolFactory", + "ETLProtocolFactoriesInitializer", + "LocationIDInitializer", + "LocationNameInitializer", + "ProtocolDefinition", +] diff --git a/app/mods/common/src/sghi/idr/client/common/lib/config/etl_protocols_config.py b/app/mods/common/src/sghi/idr/client/common/lib/config/etl_protocols_config.py new file mode 100644 index 0000000..cb69be5 --- /dev/null +++ b/app/mods/common/src/sghi/idr/client/common/lib/config/etl_protocols_config.py @@ -0,0 +1,212 @@ +from collections.abc import Callable, Iterable, Sequence +from functools import cache +from typing import Any, Final, TypedDict + +from sghi.idr.client.core.domain import ( + DataSink, + DataSinkMetadata, + DataSource, + DataSourceMetadata, + ETLProtocol, + ExtractProcessor, + MetadataConsumer, + MetadataSupplier, + UploadMetadataFactory, +) +from sghi.idr.client.core.lib import ( + ImproperlyConfiguredError, + SettingInitializer, + import_string, +) +from typing_inspect import typed_dict_keys + +# ============================================================================= +# TYPES +# ============================================================================= + + +ETLProtocolFactory = Callable[[], ETLProtocol] + + +class _ProtocolDefinitionOptional(TypedDict, total=False): + description: str | None + + +class _RawProtocolDefinition(_ProtocolDefinitionOptional, total=True): + id: str # noqa: A003 + name: str + data_sink_factory: str # Callable[[DataSinkMetadata], DataSink] + data_source_factory: str # Callable[[DataSourceMetadata], DataSource] + extract_processor_factory: str # Callable[[], ExtractProcessor] + metadata_consumer_factory: str # Callable[[], MetadataConsumer] + metadata_supplier_factory: str # Callable[[], MetadataSupplier] + upload_metadata_factory: str # UploadMetadataFactory + + +class ProtocolDefinition(_ProtocolDefinitionOptional, total=True): + id: str # noqa: A003 + name: str + data_sink_factory: Callable[[DataSinkMetadata], DataSink] + data_source_factory: Callable[[DataSourceMetadata], DataSource] + extract_processor_factory: Callable[[], ExtractProcessor] + metadata_consumer_factory: Callable[[], MetadataConsumer] + metadata_supplier_factory: Callable[[], MetadataSupplier] + upload_metadata_factory: UploadMetadataFactory | Callable[ + [], + UploadMetadataFactory, + ] + + +# ============================================================================= +# CONSTANTS +# ============================================================================= + +_UNKNOWN_STR: Final[str] = "UNKNOWN" + +ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY: Final[str] = "ETL_PROTOCOL_DEFINITIONS" + +ETL_PROTOCOL_FACTORIES_CONFIG_KEY: Final[str] = "ETL_PROTOCOL_FACTORIES" + + +# ============================================================================= +# HELPERS +# ============================================================================= + + +@cache +def _get_required_proto_definition_fields() -> set[str]: + all_fields: set[str] = set( + typed_dict_keys(_RawProtocolDefinition).keys(), + ) + optional_fields: set[str] = set( + typed_dict_keys(_ProtocolDefinitionOptional).keys(), + ) + return all_fields.difference(optional_fields) + + +# ============================================================================= +# SETTINGS INITIALIZERS +# ============================================================================= + + +class ETLProtocolDefinitionsInitializer(SettingInitializer): + """ + :class:`SettingInitializer` that loads :class:`ETLProtocol` definitions for + later instantiation. + """ + + @property + def has_secrets(self) -> bool: + return False + + @property + def setting(self) -> str: + return ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY + + def execute( + self, + an_input: Iterable[_RawProtocolDefinition] | None, + ) -> Iterable[ProtocolDefinition]: + return tuple(map(self._mapping_to_definition, an_input or ())) + + def _mapping_to_definition( + self, + raw_protocol_definition: _RawProtocolDefinition, + ) -> ProtocolDefinition: + self._check_raw_definition(raw_protocol_definition) + return { + "id": raw_protocol_definition["id"], + "name": raw_protocol_definition["name"], + "description": raw_protocol_definition.get("description"), + "data_sink_factory": self._load_entry_point( + raw_protocol_definition["data_sink_factory"], + item_name="data_sink_factory", + ), + "data_source_factory": self._load_entry_point( + raw_protocol_definition["data_source_factory"], + item_name="data_source_factory", + ), + "extract_processor_factory": self._load_entry_point( + raw_protocol_definition["extract_processor_factory"], + item_name="extract_processor_factory", + ), + "metadata_consumer_factory": self._load_entry_point( + raw_protocol_definition["metadata_consumer_factory"], + item_name="metadata_consumer_factory", + ), + "metadata_supplier_factory": self._load_entry_point( + raw_protocol_definition["metadata_supplier_factory"], + item_name="metadata_supplier_factory", + ), + "upload_metadata_factory": self._load_entry_point( + raw_protocol_definition["upload_metadata_factory"], + item_name="upload_metadata_factory", + ), + } + + @staticmethod + def _check_raw_definition( + raw_protocol_definition: _RawProtocolDefinition, + ) -> None: + required_fields: set[str] = _get_required_proto_definition_fields() + missing_fields: set[str] = required_fields.difference( + set(raw_protocol_definition.keys()), + ) + if len(missing_fields) > 0: + _err_msg: str = ( + "The following missing fields must be provided for each " + "protocol definition: '{}'.".format(",".join(missing_fields)) + ) + raise ImproperlyConfiguredError(message=_err_msg) + + @staticmethod + def _load_entry_point( + item_dotted_path: str, + item_name: str, + ) -> Any: # noqa: ANN401 + try: + return import_string(dotted_path=item_dotted_path) + except ImportError as exp: + _err_msg: str = ( + "Invalid {} given. '{}' does not seem to be a valid path." + .format(item_name, item_dotted_path) + ) + raise ImproperlyConfiguredError(message=_err_msg) from exp + + +class ETLProtocolFactoriesInitializer(SettingInitializer): + """ + :class:`SettingInitializer` that loads :class:`ETLProtocol` factories for + later instantiation. + """ + + @property + def has_secrets(self) -> bool: + return False + + @property + def setting(self) -> str: + return ETL_PROTOCOL_FACTORIES_CONFIG_KEY + + def execute( + self, + an_input: Sequence[str] | None, + ) -> Sequence[ETLProtocolFactory]: + protocol_factories: Sequence[ETLProtocolFactory] = list( + map(self._dotted_path_to_factory, an_input or ()), + ) + return protocol_factories + + @staticmethod + def _dotted_path_to_factory( + protocol_dotted_path: str, + ) -> ETLProtocolFactory: + try: + etl_protocol_factory: ETLProtocolFactory + etl_protocol_factory = import_string(protocol_dotted_path) + return etl_protocol_factory + except ImportError as exp: + _err_msg: str = '"{}" does not seem to be a valid path.'.format( + protocol_dotted_path, + ) + raise ImproperlyConfiguredError(message=_err_msg) from exp diff --git a/app/mods/common/src/sghi/idr/client/common/lib/config.py b/app/mods/common/src/sghi/idr/client/common/lib/config/location_config.py similarity index 100% rename from app/mods/common/src/sghi/idr/client/common/lib/config.py rename to app/mods/common/src/sghi/idr/client/common/lib/config/location_config.py diff --git a/app/mods/idr_server/pyproject.toml b/app/mods/idr_server/pyproject.toml index d979f66..a47ac9c 100644 --- a/app/mods/idr_server/pyproject.toml +++ b/app/mods/idr_server/pyproject.toml @@ -46,6 +46,9 @@ name = "sghi-idr-client-idr-server" readme = "README.md" requires-python = ">=3.10" # Support Python 3.10+. +[project.entry-points."sghi.idr.client.etl_protocol"] +idr_server = "sghi.idr.client.idr_server:IDRServerETLProtocolSupplier" + [project.urls] changelog = "https://github.com/savannahghi/idr-client/blob/develop/docs/CHANGELOG.md" documentation = "https://github.com/savannahghi/idr-client/blob/develop/README.md" diff --git a/app/mods/idr_server/src/sghi/idr/client/idr_server/__init__.py b/app/mods/idr_server/src/sghi/idr/client/idr_server/__init__.py index a378524..71036ba 100644 --- a/app/mods/idr_server/src/sghi/idr/client/idr_server/__init__.py +++ b/app/mods/idr_server/src/sghi/idr/client/idr_server/__init__.py @@ -1,4 +1,7 @@ +from collections.abc import Iterable + from sghi.idr.client.common.domain import SimpleETLProtocol +from sghi.idr.client.core.domain import ETLProtocolSupplier from sghi.idr.client.http import ( HTTPDataSink, HTTPMetadataConsumer, @@ -34,31 +37,65 @@ ] +# ============================================================================= +# HELPERS +# ============================================================================= + + def fyj_cbs_etl_protocol_factory() -> FYJCBSETLProtocol: return SimpleETLProtocol( id="fyj-cbs", - name="FYJ CBS ETL Protocol", + name="FyJ CBS ETL Protocol", description="Fahari ya Jamii, CBS ETL Protocol", data_sink_factory=HTTPDataSink.from_data_sink_meta, data_source_factory=SimpleSQLDatabase.from_data_source_meta, extract_processor_factory=IDRServerExtractProcessor, - metadata_consumer=HTTPMetadataConsumer( - name="FyJ IDR Server Metadata Consumer", # pyright: ignore - api_dialect=idr_server_api_factory(), # pyright: ignore - transport=http_transport_factory(), # pyright: ignore - ), - metadata_supplier=HTTPMetadataSupplier( # pyright: ignore - name="FyJ IDR Server Metadata Supplier", # pyright: ignore - api_dialect=idr_server_api_factory(), # pyright: ignore - transport=http_transport_factory(), # pyright: ignore - ), - upload_metadata_factory=HTTPUploadMetadataFactory( # pyright: ignore - api_dialect=idr_server_api_factory(), # pyright: ignore - transport=http_transport_factory(), # pyright: ignore - ), + metadata_consumer=fyj_cbs_metadata_consumer_factory(), + metadata_supplier=fyj_cbs_metadata_supplier_factory(), + upload_metadata_factory=fyj_cbs_upload_meta_factory(), + ) + + +def fyj_cbs_metadata_consumer_factory() -> HTTPMetadataConsumer: + return HTTPMetadataConsumer( + name="FyJ IDR Server Metadata Consumer", # pyright: ignore + api_dialect=idr_server_api_factory(), # pyright: ignore + transport=http_transport_factory(), # pyright: ignore + ) + + +def fyj_cbs_metadata_supplier_factory() -> HTTPMetadataSupplier: + return HTTPMetadataSupplier( + name="FyJ IDR Server Metadata Supplier", # pyright: ignore + api_dialect=idr_server_api_factory(), # pyright: ignore + transport=http_transport_factory(), # pyright: ignore ) -__all__ = ["fyj_cbs_etl_protocol_factory"] +def fyj_cbs_upload_meta_factory() -> HTTPUploadMetadataFactory: + return HTTPUploadMetadataFactory( + api_dialect=idr_server_api_factory(), # pyright: ignore + transport=http_transport_factory(), # pyright: ignore + ) + + +# ============================================================================= +# ETL PROTOCOL SUPPLIERS +# ============================================================================= + + +class IDRServerETLProtocolSupplier(ETLProtocolSupplier): + def get_protocols( + self, + ) -> Iterable[FYJCBSETLProtocol]: + return (fyj_cbs_etl_protocol_factory(),) + + +__all__ = [ + "fyj_cbs_etl_protocol_factory", + "fyj_cbs_metadata_consumer_factory", + "fyj_cbs_metadata_supplier_factory", + "fyj_cbs_upload_meta_factory", +] __all__ += _all_domain # type: ignore __all__ += _all_lib # type: ignore diff --git a/app/runtime/src/sghi/idr/client/runtime/constants.py b/app/runtime/src/sghi/idr/client/runtime/constants.py index 08a99b5..fd2aa16 100644 --- a/app/runtime/src/sghi/idr/client/runtime/constants.py +++ b/app/runtime/src/sghi/idr/client/runtime/constants.py @@ -1,17 +1,18 @@ from typing import Any, Final -APP_DISPATCHER_REG_KEY: Final[str] = "runtime.app_dispatcher" +APP_DISPATCHER_REG_KEY: Final[str] = "sghi.idr.client.runtime.app_dispatcher" APP_VERBOSITY_REG_KEY: Final[str] = "runtime.verbosity" -ETL_PROTOCOLS_CONFIG_KEY: Final[str] = "ETL_PROTOCOLS" +ETL_PROTOCOLS_ENTRY_POINT_GROUP_NAME: Final[ + str +] = "sghi.idr.client.etl_protocol" LOGGING_CONFIG_KEY: Final[str] = "LOGGING" SETTINGS_INITIALIZERS_CONFIG_KEY: Final[str] = "SETTINGS_INITIALIZERS" DEFAULT_CONFIG: Final[dict[str, Any]] = { - ETL_PROTOCOLS_CONFIG_KEY: [], LOGGING_CONFIG_KEY: { "version": 1, "formatters": { diff --git a/app/runtime/src/sghi/idr/client/runtime/settings_initializers.py b/app/runtime/src/sghi/idr/client/runtime/settings_initializers.py index c74d8c7..c820b19 100644 --- a/app/runtime/src/sghi/idr/client/runtime/settings_initializers.py +++ b/app/runtime/src/sghi/idr/client/runtime/settings_initializers.py @@ -1,57 +1,12 @@ import logging -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from logging.config import dictConfig from typing import Any import sghi.idr.client.core as app -from sghi.idr.client.core.lib import ( - ImproperlyConfiguredError, - SettingInitializer, - import_string, -) +from sghi.idr.client.core.lib import SettingInitializer -from .constants import ( - DEFAULT_CONFIG, - ETL_PROTOCOLS_CONFIG_KEY, - LOGGING_CONFIG_KEY, -) -from .typings import ETLProtocol_Factory - - -class ETLProtocolInitializer(SettingInitializer): - """ - :class:`SettingInitializer` that loads :class:`ETLProtocol` factories for - later instantiation. - """ - - @property - def setting(self) -> str: - return ETL_PROTOCOLS_CONFIG_KEY - - def execute( - self, - an_input: Sequence[str], - ) -> Sequence[ETLProtocol_Factory]: - etl_protocols: Sequence[ETLProtocol_Factory] = list( - map(self._dotted_path_to_etl_protocol_factory, an_input), - ) - return etl_protocols - - @staticmethod - def _dotted_path_to_etl_protocol_factory( - etl_protocol_dotted_path: str, - ) -> ETLProtocol_Factory: - try: - etl_protocol_factory: ETLProtocol_Factory - etl_protocol_factory = import_string( # type: ignore - etl_protocol_dotted_path, - ) - return etl_protocol_factory - except ImportError as exp: - _err_msg: str = '"{}" does not seem to be a valid path.'.format( - etl_protocol_dotted_path, - ) - raise ImproperlyConfiguredError(message=_err_msg) from exp +from .constants import DEFAULT_CONFIG, LOGGING_CONFIG_KEY class LoggingInitializer(SettingInitializer): diff --git a/app/runtime/src/sghi/idr/client/runtime/setup.py b/app/runtime/src/sghi/idr/client/runtime/setup.py index af742f1..70c5215 100644 --- a/app/runtime/src/sghi/idr/client/runtime/setup.py +++ b/app/runtime/src/sghi/idr/client/runtime/setup.py @@ -2,22 +2,16 @@ from typing import Any, cast import sghi.idr.client.core as app -from sghi.idr.client.core.domain import ETLProtocol from sghi.idr.client.core.lib import ( Config, ImproperlyConfiguredError, SettingInitializer, import_string_as_klass, ) -from toolz import first, pipe -from toolz.curried import groupby, map, valmap +from toolz import pipe +from toolz.curried import map -from .constants import ( - DEFAULT_CONFIG, - ETL_PROTOCOLS_CONFIG_KEY, - SETTINGS_INITIALIZERS_CONFIG_KEY, -) -from .typings import ETLProtocol_Factory +from .constants import DEFAULT_CONFIG, SETTINGS_INITIALIZERS_CONFIG_KEY # ============================================================================= # HELPERS @@ -49,38 +43,6 @@ def _dotted_path_to_initializer_instance( raise ImproperlyConfiguredError(message=_err_msg) from exp -def _etl_protocol_factory_to_instance( - _etl_protocol_factory: ETLProtocol_Factory, -) -> ETLProtocol: - _etl_protocol_instance: ETLProtocol = _etl_protocol_factory() - if not isinstance(_etl_protocol_instance, ETLProtocol): - _err_msg: str = ( - 'Invalid ETLProtocol, the factory "{}.{}" returned an instance ' - 'that is not a subclass of "app.core.domain.ETLProtocol".'.format( - # noinspection PyUnresolvedReferences - _etl_protocol_factory.__module__, - _etl_protocol_factory.__qualname__, - ) - ) - raise ImproperlyConfiguredError(message=_err_msg) - - return _etl_protocol_instance - - -def _initialize_and_load_etl_protocols( - etl_protocol_factories: Sequence[ETLProtocol_Factory], -) -> None: - app.registry.etl_protocols = cast( - Mapping[str, ETLProtocol], - pipe( - etl_protocol_factories, - map(_etl_protocol_factory_to_instance), - groupby(lambda _ep: _ep.id), - valmap(first), - ), - ) - - def load_settings_initializers( initializers_dotted_paths: Sequence[str], ) -> Sequence[SettingInitializer]: @@ -133,19 +95,12 @@ def setup( ) if not disable_default_initializers: - from .settings_initializers import ( - ETLProtocolInitializer, - LoggingInitializer, - ) + from .settings_initializers import LoggingInitializer initializers.insert(0, LoggingInitializer()) - initializers.insert(1, ETLProtocolInitializer()) app.registry.log_level = log_level # noinspection app.settings = Config( settings=settings_dict, settings_initializers=initializers, ) - _initialize_and_load_etl_protocols( - app.settings.get(ETL_PROTOCOLS_CONFIG_KEY, []), - ) diff --git a/app/runtime/src/sghi/idr/client/runtime/tui/rich/ui.py b/app/runtime/src/sghi/idr/client/runtime/tui/rich/ui.py index e661d95..dcf7069 100644 --- a/app/runtime/src/sghi/idr/client/runtime/tui/rich/ui.py +++ b/app/runtime/src/sghi/idr/client/runtime/tui/rich/ui.py @@ -1,7 +1,6 @@ from attrs import field, frozen from rich.console import Console from rich.live import Live -from rich.status import Status from sghi.idr.client.runtime.constants import APP_DISPATCHER_REG_KEY from sghi.idr.client.runtime.ui import UI from sghi.idr.client.runtime.utils import dispatch @@ -14,27 +13,13 @@ def _app_live_display_factory(console: Console = CONSOLE) -> Live: return Live(console=console, refresh_per_second=12.5) -def _app_run_status_factory(console: Console = CONSOLE) -> Status: - return Status( - "[bold]Running ...", - console=console, - spinner="dots", - spinner_style="bright_yellow", - ) - - @frozen class RichUI(UI): """:class:`UI` implementation that uses a :class:`rich.console.Console` to render output on the console. """ - _app_run_status: Status = field( - factory=_app_run_status_factory, - init=False, - ) _console: Console = field(default=CONSOLE, init=False) - _etl_proto_statuses: dict[str, Status] = field(factory=dict, init=False) _etl_proto_uis: dict[str, ETLProtocolUI] = field(factory=dict, init=False) _live_display: Live = field(factory=_app_live_display_factory, init=False) @@ -67,6 +52,10 @@ def start(self) -> None: dispatch.PreETLWorkflowRunSignal, self.on_etl_workflow_start, ) + app_dispatcher.connect( + dispatch.UnhandledRuntimeErrorSignal, + self.on_runtime_error, + ) def on_app_start(self, signal: dispatch.AppPreStartSignal) -> None: self._console.log("[bold cyan]Starting ... ") @@ -123,3 +112,9 @@ def on_etl_workflow_stop( self._etl_proto_uis[signal.etl_protocol.id].stop_workflow( extract_meta=signal.extract_meta, ) + + def on_runtime_error( + self, + signal: dispatch.UnhandledRuntimeErrorSignal, + ) -> None: + self._live_display.stop() diff --git a/app/runtime/src/sghi/idr/client/runtime/usecases/__init__.py b/app/runtime/src/sghi/idr/client/runtime/usecases/__init__.py index 4be792c..5255cc1 100644 --- a/app/runtime/src/sghi/idr/client/runtime/usecases/__init__.py +++ b/app/runtime/src/sghi/idr/client/runtime/usecases/__init__.py @@ -1,14 +1,26 @@ +from typing import TYPE_CHECKING + import sghi.idr.client.core as app -from sghi.idr.client.runtime.constants import APP_DISPATCHER_REG_KEY +from importlib_metadata import EntryPoint, entry_points +from sghi.idr.client.runtime.constants import ( + APP_DISPATCHER_REG_KEY, + ETL_PROTOCOLS_ENTRY_POINT_GROUP_NAME, +) from sghi.idr.client.runtime.utils import dispatch +from toolz.curried import groupby, map, valmap +from toolz.functoolz import pipe +from toolz.itertoolz import concat, first from .etl_workflow import ETLWorkflow from .run_etl_protocol import RunETLProtocol +if TYPE_CHECKING: + from collections.abc import Callable + + from sghi.idr.client.core.domain import ETLProtocol, ETLProtocolSupplier -def start() -> None: - """Entry point for running all use-cases.""" +def _execute_protocols() -> None: app_dispatcher: dispatch.Dispatcher app_dispatcher = app.registry.get(APP_DISPATCHER_REG_KEY) for etl_protocol in app.registry.etl_protocols.values(): @@ -17,6 +29,33 @@ def start() -> None: app_dispatcher.send(dispatch.PostETLProtocolRunSignal(etl_protocol)) +def _load_etl_protocols() -> None: + # TODO: Improve this, check that invariants are upheld, etc. + _ep: EntryPoint + _eps: ETLProtocolSupplier + _eps_factory: Callable[[], ETLProtocolSupplier] + _etl_proto: ETLProtocol + app.registry.etl_protocols = pipe( + entry_points(group=ETL_PROTOCOLS_ENTRY_POINT_GROUP_NAME), + map(lambda _ep: _ep.load()), + map(lambda _eps_factory: _eps_factory()), + map(lambda _eps: _eps.get_protocols()), + concat, + groupby(lambda _etl_proto: _etl_proto.id), + valmap(first), + ) + + +def start() -> None: + """Entry point for running all use-cases. + + Load and execute etl protocols. + """ + + _load_etl_protocols() + _execute_protocols() + + __all__ = [ "ETLWorkflow", "RunETLProtocol",