Add a new core interface ETLProtocolSupplier whose purpose is to su…

…pply

`ETLProtocol` instances. As a result of this, add `ETLProtocolSupplier`
implementations on some of the mods such as `common` and `idr_server`.
The implementations on the `common` mod serve as utilities that add the
ability to load `ETLProtocol` instances defined on the config either as
factory functions that return said instances or either as a dictionary
of key value pairs that can be assembeled into an `ETLProtocol`
instance. The implementation on the `idr_server` mod returns `ETLProtocol`
instances that work with the idr_server such as the fahari ya jamii cbs
protocol.

The `runtime` mod has also being refactored to allow the loading of
`ETLProtocol` instances as package entry points. The import functions
on the `module_loading` package have also being refactored to use the
`EntryPoint.load` method to load new modules.

.pre-commit-config.yaml

@@ -17,17 +17,6 @@ repos:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]
 
-  - repo: https://github.com/psf/black
-    rev: 23.3.0
-    hooks:
-      - id: black
-
-  - repo: https://github.com/timothycrosley/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        args: ["--profile", "black"]
-
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.3.1
     hooks:

app/core/src/sghi/idr/client/core/domain/interfaces/__init__.py

@@ -1,5 +1,5 @@
 from .base import DomainObject, IdentifiableDomainObject, NamedDomainObject
-from .etl_protocol import ETLProtocol
+from .etl_protocol import ETLProtocol, ETLProtocolSupplier
 from .metadata import (
     DataSinkMetadata,
     DataSourceMetadata,
@@ -35,6 +35,7 @@
     "DataSourceMetadata",
     "DomainObject",
     "ETLProtocol",
+    "ETLProtocolSupplier",
     "ExtractMetadata",
     "ExtractProcessor",
     "IdentifiableDomainObject",

app/core/src/sghi/idr/client/core/domain/interfaces/etl_protocol.py

@@ -2,8 +2,8 @@
 ETL Protocol Definition.
 """
 from abc import ABCMeta, abstractmethod
-from collections.abc import Callable
-from typing import Generic, TypeVar
+from collections.abc import Callable, Iterable
+from typing import Any, Generic, TypeVar
 
 from .base import IdentifiableDomainObject, NamedDomainObject
 from .operations import (
@@ -83,3 +83,11 @@ def metadata_supplier(self) -> MetadataSupplier[_DS, _DM, _EM]:
     @abstractmethod
     def upload_metadata_factory(self) -> UploadMetadataFactory[_UM, _EM]:
         ...
+
+
+class ETLProtocolSupplier(metaclass=ABCMeta):
+    @abstractmethod
+    def get_protocols(
+        self,
+    ) -> Iterable[ETLProtocol[Any, Any, Any, Any, Any, Any]]:
+        ...

app/core/src/sghi/idr/client/core/lib/module_loading.py

@@ -1,9 +1,7 @@
-# The contents of this module are copied from Django sources.
 import inspect
-import sys
-from importlib import import_module
-from types import ModuleType
-from typing import TypeVar, cast
+from typing import Any, Final, TypeVar, cast
+
+from importlib_metadata import EntryPoint
 
 # =============================================================================
 # TYPES
@@ -13,54 +11,43 @@
 
 
 # =============================================================================
-# HELPERS
+# CONSTANTS
 # =============================================================================
 
-
-def _cached_import(module_path: str, class_name: str) -> ModuleType:
-    modules = sys.modules
-    if module_path not in modules or (
-        # Module is not fully initialized.
-        getattr(modules[module_path], "__spec__", None) is not None
-        and getattr(modules[module_path].__spec__, "_initializing", False)
-        is True
-    ):  # pragma: no branch
-        import_module(module_path)
-    return getattr(modules[module_path], class_name)
+_UNKNOWN_STR: Final[str] = "UNKNOWN"
 
 
 # =============================================================================
 # IMPORT UTILITIES
 # =============================================================================
 
 
-def import_string(dotted_path: str) -> ModuleType:
+def import_string(dotted_path: str) -> Any:  # noqa: ANN401
     """
     Import a dotted module path and return the attribute/class designated by
     the last name in the path. Raise ``ImportError`` if the import failed.
 
+    The `dotted_path` should conform to the format defined by the Python
+    packaging conventions. See `the packaging docs on entry points
+    <https://packaging.python.org/specifications/entry-points/>`_
+    for more information.
+
     :param dotted_path: A dotted path to an attribute or class.
 
     :return: The attribute/class designated by the last name in the path.
 
     :raise ImportError: If the import fails for some reason.
     """
+    entry_point = EntryPoint(
+        name=_UNKNOWN_STR,
+        group=_UNKNOWN_STR,
+        value=dotted_path,
+    )
     try:
-        module_path, class_name = dotted_path.rsplit(".", 1)
-    except ValueError as err:
-        err_msg: str = f'"{dotted_path}" does not look like a module path.'
-        raise ImportError(err_msg, path=dotted_path) from err
-
-    try:
-        return _cached_import(module_path, class_name)
-    except AttributeError as err:
-        err_msg: str = (
-            'Module "{}" does not define a "{}" attribute/class'.format(
-                module_path,
-                class_name,
-            )
-        )
-        raise ImportError(err_msg, name=class_name, path=module_path) from err
+        return entry_point.load()
+    except AttributeError as exp:
+        _err_msg: str = str(exp)
+        raise ImportError(_err_msg) from exp
 
 
 def import_string_as_klass(

app/mods/common/pyproject.toml

@@ -39,6 +39,10 @@ name = "sghi-idr-client-common"
 readme = "README.md"
 requires-python = ">=3.10" # Support Python 3.10+.
 
+[project.entry-points."sghi.idr.client.etl_protocol"]
+from_definitions = "sghi.idr.client.common.domain:FromDefinitionsETLProtocolSupplier"
+from_factories = "sghi.idr.client.common.domain:FromFactoriesETLProtocolSupplier"
+
 [project.urls]
 changelog = "https://github.com/savannahghi/idr-client/blob/develop/docs/CHANGELOG.md"
 documentation = "https://github.com/savannahghi/idr-client/blob/develop/README.md"

app/mods/common/src/sghi/idr/client/common/domain/__init__.py

@@ -1,5 +1,11 @@
-from .etl_protocol import SimpleETLProtocol
+from .etl_protocol import (
+    FromDefinitionsETLProtocolSupplier,
+    FromFactoriesETLProtocolSupplier,
+    SimpleETLProtocol,
+)
 
 __all__ = [
+    "FromDefinitionsETLProtocolSupplier",
+    "FromFactoriesETLProtocolSupplier",
     "SimpleETLProtocol",
 ]

app/mods/common/src/sghi/idr/client/common/domain/etl_protocol.py

@@ -1,5 +1,5 @@
-from collections.abc import Callable
-from typing import Generic, TypeVar
+from collections.abc import Callable, Iterable
+from typing import Any, Generic, TypeVar, assert_never
 
 from attrs import define, field
 from sghi.idr.client.core.domain import (
@@ -11,6 +11,7 @@
     DataSource,
     DataSourceMetadata,
     ETLProtocol,
+    ETLProtocolSupplier,
     ExtractMetadata,
     ExtractProcessor,
     MetadataConsumer,
@@ -19,6 +20,14 @@
     UploadMetadata,
     UploadMetadataFactory,
 )
+from sghi.idr.client.core.lib import ImproperlyConfiguredError, type_fqn
+
+from ..lib import (
+    ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY,
+    ETL_PROTOCOL_FACTORIES_CONFIG_KEY,
+    ETLProtocolFactory,
+    ProtocolDefinition,
+)
 
 # =============================================================================
 # TYPES
@@ -31,9 +40,15 @@
 _RD = TypeVar("_RD", bound=RawData)
 _UM = TypeVar("_UM", bound=UploadMetadata)
 
+_EP = ETLProtocol[Any, Any, Any, Any, Any, Any]
+
 
 # =============================================================================
-# CONCRETE ETL PROTOCOL DEFINITION
+# HELPERS
+# =============================================================================
+
+# =============================================================================
+# CONCRETE ETL PROTOCOL DEFINITIONS
 # =============================================================================
 
 
@@ -57,11 +72,13 @@ class SimpleETLProtocol(
     _upload_metadata_factory: UploadMetadataFactory[_UM, _EM] = field()
 
     @property
-    def data_sink_factory(self) -> Callable[[_DS], DataSink]:
+    def data_sink_factory(self) -> Callable[[_DS], DataSink[_DS, _UM, _CD]]:
         return self._data_sink_factory
 
     @property
-    def data_source_factory(self) -> Callable[[_DM], DataSource]:
+    def data_source_factory(
+        self,
+    ) -> Callable[[_DM], DataSource[_DM, _EM, _RD]]:
         return self._data_source_factory
 
     @property
@@ -81,3 +98,109 @@ def metadata_supplier(self) -> MetadataSupplier[_DS, _DM, _EM]:
     @property
     def upload_metadata_factory(self) -> UploadMetadataFactory[_UM, _EM]:
         return self._upload_metadata_factory
+
+
+# =============================================================================
+# ETL PROTOCOL SUPPLIERS
+# =============================================================================
+
+
+class FromDefinitionsETLProtocolSupplier(ETLProtocolSupplier):
+    """
+    Load :class:`ETLProtocol` instances from ETLProtocol definitions on the
+    config.
+    """
+
+    def get_protocols(self) -> Iterable[_EP]:
+        from sghi.idr.client.core import settings
+
+        proto_definitions: Iterable[ProtocolDefinition]
+        proto_definitions = settings.get(
+            setting=ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY,
+            default=(),
+        )
+        return map(
+            self._proto_definition_to_proto_instance,
+            proto_definitions,
+        )
+
+    @classmethod
+    def _proto_definition_to_proto_instance(
+        cls,
+        protocol_definition: ProtocolDefinition,
+    ) -> _EP:
+        upf_def = protocol_definition["upload_metadata_factory"]
+        upf: UploadMetadataFactory = cls._get_upload_meta_factory_instance(
+            upf_def,
+        )
+        return SimpleETLProtocol(
+            id=protocol_definition["id"],
+            name=protocol_definition["name"],
+            description=protocol_definition.get("description"),
+            data_sink_factory=protocol_definition["data_sink_factory"],
+            data_source_factory=protocol_definition["data_source_factory"],
+            extract_processor_factory=protocol_definition[
+                "extract_processor_factory"
+            ],
+            metadata_consumer=protocol_definition[
+                "metadata_consumer_factory"
+            ](),
+            metadata_supplier=protocol_definition[
+                "metadata_supplier_factory"
+            ](),
+            upload_metadata_factory=upf,
+        )
+
+    @staticmethod
+    def _get_upload_meta_factory_instance(
+        upload_meta_factory: UploadMetadataFactory
+        | Callable[[], UploadMetadataFactory],
+    ) -> UploadMetadataFactory:
+        match upload_meta_factory:
+            case UploadMetadataFactory():
+                return upload_meta_factory
+            case Callable():
+                return upload_meta_factory()
+            case _:
+                assert_never(upload_meta_factory)
+
+
+class FromFactoriesETLProtocolSupplier(ETLProtocolSupplier):
+    """
+    Load :class:`ETLProtocol` instances from ETLProtocol factories on the
+    config.
+    """
+
+    def get_protocols(self) -> Iterable[_EP]:
+        from sghi.idr.client.core import settings
+
+        proto_factories: Iterable[ETLProtocolFactory]
+        proto_factories = settings.get(
+            setting=ETL_PROTOCOL_FACTORIES_CONFIG_KEY,
+            default=(),
+        )
+
+        return map(self._proto_factory_to_instance, proto_factories)
+
+    @staticmethod
+    def _proto_factory_to_instance(proto_factory: ETLProtocolFactory) -> _EP:
+        try:
+            _etl_proto_instance: _EP = proto_factory()
+        except Exception as exp:  # noqa: BLE001
+            _err_msg: str = (
+                "Unable to create an ETLProtocol instance from factory. The "
+                "cause was: '{}'".format(str(exp))
+            )
+            raise RuntimeError(_err_msg) from exp
+
+        if not isinstance(_etl_proto_instance, ETLProtocol):
+            _err_msg: str = (
+                "Invalid ETLProtocol, the factory '{}' returned an instance "
+                "that is not a subclass of "
+                "'app.core.domain.ETLProtocol'.".format(
+                    type_fqn(proto_factory),
+                )
+            )
+            raise ImproperlyConfiguredError(message=_err_msg)
+
+        return _etl_proto_instance

app/mods/common/src/sghi/idr/client/common/lib/__init__.py

@@ -1,6 +1,5 @@
-from .config import LocationIDInitializer, LocationNameInitializer
+from .config import *  # noqa: F403
+from .config import __all__ as _all_config
 
-__all__ = [
-    "LocationIDInitializer",
-    "LocationNameInitializer",
-]
+__all__ = []
+__all__ += _all_config  # type: ignore

app/mods/common/src/sghi/idr/client/common/lib/config/__init__.py

@@ -0,0 +1,20 @@
+from .etl_protocols_config import (
+    ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY,
+    ETL_PROTOCOL_FACTORIES_CONFIG_KEY,
+    ETLProtocolDefinitionsInitializer,
+    ETLProtocolFactoriesInitializer,
+    ETLProtocolFactory,
+    ProtocolDefinition,
+)
+from .location_config import LocationIDInitializer, LocationNameInitializer
+
+__all__ = [
+    "ETL_PROTOCOL_DEFINITIONS_CONFIG_KEY",
+    "ETL_PROTOCOL_FACTORIES_CONFIG_KEY",
+    "ETLProtocolDefinitionsInitializer",
+    "ETLProtocolFactory",
+    "ETLProtocolFactoriesInitializer",
+    "LocationIDInitializer",
+    "LocationNameInitializer",
+    "ProtocolDefinition",
+]