refactor(domain): domain models to simplify the codebase (#29)

app/__version__.py

@@ -2,6 +2,6 @@
 __description__ = "Just Another ETL Tool."
 __url__ = "https://github.com/savannahghi/idr-client"
 __version__ = "1.0.0"
-__author__ = "Savannah Informatics Global Health Initiative"
+__author__ = "Savannah Informatics Global Health Institute"
 __license__ = "MIT"
-__copyright__ = "Copyright 2022 Savannah Informatics Global Health Initiative"
+__copyright__ = "Copyright 2022 Savannah Informatics Global Health Institute"

app/core/domain.py

@@ -137,6 +137,17 @@ class ExtractMetadata(
     description: Optional[str]
     preferred_uploads_name: Optional[str]
 
+    @property
+    @abstractmethod
+    def data_source(self) -> "DataSource":
+        """
+        Return the :class:`data source <DataSource>` that contains this extract
+        metadata.
+
+        :return: The data source that contains this extract metadata.
+        """
+        ...
+
     def __str__(self) -> str:
         return "%s::%s" % (self.id, self.name)
 
@@ -167,6 +178,17 @@ class DataSource(
     name: str
     description: Optional[str]
 
+    @property
+    @abstractmethod
+    def data_source_type(self) -> "DataSourceType":
+        """
+        Return the :class:`data source type <DataSourceType>` that this data
+        source belongs to.
+
+        :return: The data source type that this data source belongs to.
+        """
+        ...
+
     @property
     @abstractmethod
     def extract_metadata(self) -> Mapping[str, ExtractMetadata[_RT, Any]]:
@@ -182,7 +204,7 @@ def extract_metadata(self) -> Mapping[str, ExtractMetadata[_RT, Any]]:
     @extract_metadata.setter
     @abstractmethod
     def extract_metadata(
-        self, extract_metadata: Mapping[str, ExtractMetadata]
+        self, extract_metadata: Mapping[str, ExtractMetadata[_RT, Any]]
     ) -> None:
         """Set the extract metadata instances that belong to this data source.
 

app/imp/sql_data/domain.py

@@ -6,7 +6,7 @@
 from sqlalchemy.engine import Connection, Engine
 
 import app
-from app.core import DataSource, DataSourceType, ExtractMetadata, ToTask
+from app.core import DataSource, DataSourceType, ExtractMetadata
 from app.lib import ImproperlyConfiguredError, SimpleSQLSelect
 
 from .exceptions import SQLDataError, SQLDataSourceDisposedError
@@ -39,7 +39,9 @@ class SQLDataSource(DataSource[Connection]):
     database_vendor: SupportedDBVendors
 
     def __init__(self, **kwargs):
+        data_source_type: SQLDataSourceType = kwargs.pop("data_source_type")
         super().__init__(**kwargs)
+        self._data_source_type: SQLDataSourceType = data_source_type
         self._extract_metadata: Dict[str, "SQLExtractMetadata"] = dict()
         self._engine: Optional[Engine] = None
 
@@ -52,6 +54,10 @@ def __enter__(self) -> "SQLDataSource":
         self.connect_to_db()
         return self
 
+    @property
+    def data_source_type(self) -> "SQLDataSourceType":
+        return self._data_source_type
+
     @property
     def extract_metadata(self) -> Mapping[str, "ExtractMetadata"]:
         return self._extract_metadata
@@ -192,10 +198,18 @@ def imp_extract_metadata_klass(cls) -> Type[ExtractMetadata]:
         return SQLExtractMetadata
 
 
-class SQLExtractMetadata(ExtractMetadata, ToTask[Connection, Any]):
+class SQLExtractMetadata(ExtractMetadata[Connection, Any]):
     sql_query: str
     applicable_source_versions: Sequence[str]
 
+    def __init__(self, **kwargs):
+        data_source: SQLDataSource = kwargs.pop("data_source")
+        super().__init__(**kwargs)
+        self._data_source = data_source
+
+    @property
+    def data_source(self) -> SQLDataSource:
+        return self._data_source
+
     def to_task(self) -> SimpleSQLSelect:
-        # TODO: Add a proper implementation here.
         return SimpleSQLSelect(self.sql_query)

app/lib/tasks/sql.py

@@ -27,7 +27,7 @@ class SQLTask(Generic[_R], Task[Connection, _R], metaclass=ABCMeta):
 
 class SimpleSQLSelect(SQLTask[pd.DataFrame]):
     """
-    A task that fetches data from a dat and returns the result as a pandas
+    A task that fetches data from a database and returns the result as a pandas
     ``DataFrame``.
     """
 

app/lib/transports/http/api_v1_dialect.py

@@ -144,7 +144,11 @@ def response_to_data_source_extracts(
                 # Process/clean the response content in preparation for data
                 # source initialization.
                 execute(
-                    lambda _r: {**_r, "applicable_source_version": tuple()}
+                    lambda _r: {
+                        **_r,
+                        "applicable_source_version": tuple(),
+                        "data_source": data_source,
+                    }
                 ).
                 # Initialize the data source.
                 execute(
@@ -190,6 +194,7 @@ def response_to_data_sources(
                     lambda _r: {
                         **_r,
                         "database_vendor": SupportedDBVendors.MYSQL,
+                        "data_source_type": data_source_type,
                     }
                 ).
                 # Initialize the data source.

tests/core/factories.py

@@ -21,10 +21,16 @@ class FakeDataSource(DataSource):
     """A fake data source."""
 
     def __init__(self, **kwargs):
+        data_source_type: DataSourceType = kwargs.pop("data_source_type")
         super().__init__(**kwargs)
+        self._data_source_type: DataSourceType = data_source_type
         self._extract_metadata: Mapping[str, ExtractMetadata] = dict()
         self._is_disposed: bool = False
 
+    @property
+    def data_source_type(self) -> DataSourceType:
+        return self._data_source_type
+
     @property
     def extract_metadata(self) -> Mapping[str, "ExtractMetadata"]:
         return self._extract_metadata
@@ -77,6 +83,15 @@ def imp_extract_metadata_klass(cls) -> Type[ExtractMetadata]:
 class FakeExtractMetadata(ExtractMetadata[Any, Any]):
     """A fake extract metadata."""
 
+    def __init__(self, **kwargs):
+        data_source: DataSource = kwargs.pop("data_source")
+        super().__init__(**kwargs)
+        self._data_source: DataSource = data_source
+
+    @property
+    def data_source(self) -> DataSource:
+        return self._data_source
+
     def to_task(self) -> Task[Any, Any]:
         return self._FakeExtractTask()
 
@@ -178,6 +193,9 @@ class FakeDataSourceFactory(DataSourceFactory):
     """A factory for creating mock data source instances."""
 
     name = factory.Sequence(lambda _n: "Fake Data Source %d" % _n)
+    data_source_type = factory.SubFactory(
+        "tests.core.factories.FakeDataSourceTypeFactory"
+    )
 
     class Meta:
         model = FakeDataSource
@@ -203,6 +221,7 @@ class FakeExtractMetadataFactory(ExtractMetadataFactory):
     preferred_uploads_name = factory.LazyAttribute(
         lambda _o: "%s" % _o.name.lower().replace(" ", "_")
     )
+    data_source = factory.SubFactory(FakeDataSource)
 
     class Meta:
         model = FakeExtractMetadata

tests/core/test_domain.py

@@ -5,7 +5,13 @@
 
 from app.core import AbstractDomainObject, IdentifiableDomainObject
 
-from .factories import FakeDataSource, FakeDataSourceType, FakeExtractMetadata
+from .factories import (
+    FakeDataSource,
+    FakeDataSourceFactory,
+    FakeDataSourceType,
+    FakeDataSourceTypeFactory,
+    FakeExtractMetadata,
+)
 
 
 class _SimpleDomainObject(AbstractDomainObject):
@@ -91,7 +97,11 @@ def test_string_representation(self) -> None:
         Assert that the default ``DataSource.__str__()`` implementation
         returns the expected value.
         """
-        data_source = FakeDataSource(id="1", name="Some data source")
+        data_source = FakeDataSource(
+            id="1",
+            name="Some data source",
+            data_source_type=FakeDataSourceTypeFactory(),
+        )
         assert str(data_source) == "1::Some data source"
 
     def test_of_mapping_class_method(self) -> None:
@@ -105,13 +115,15 @@ def test_of_mapping_class_method(self) -> None:
                 "id": "1",
                 "name": "Some data source",
                 "description": "A very good description.",
+                "data_source_type": FakeDataSourceTypeFactory(),
             }
         )
         data_source2 = FakeDataSource.of_mapping(
             {
                 "id": "2",
                 "name": "Some other data source",
                 "preferred_uploads_name": "some_data",
+                "data_source_type": FakeDataSourceTypeFactory(),
             }
         )
 
@@ -145,7 +157,9 @@ def test_string_representation(self) -> None:
         Assert that the default ``ExtractMetadata.__str__()`` implementation
         returns the expected value.
         """
-        extract = FakeExtractMetadata(id="1", name="Some data")
+        extract = FakeExtractMetadata(
+            id="1", name="Some data", data_source=FakeDataSourceFactory()
+        )
         assert str(extract) == "1::Some data"
 
     def test_of_mapping_class_method(self) -> None:
@@ -159,13 +173,15 @@ def test_of_mapping_class_method(self) -> None:
                 "id": "1",
                 "name": "Some data",
                 "description": "A very good description.",
+                "data_source": FakeDataSourceFactory(),
             }
         )
         extract2 = FakeExtractMetadata.of_mapping(
             {
                 "id": "2",
                 "name": "Some other data",
                 "preferred_uploads_name": "some_data",
+                "data_source": FakeDataSourceFactory(),
             }
         )
 

tests/imp/sql_data/factories.py

@@ -20,6 +20,9 @@ class SQLDataSourceFactory(DataSourceFactory):
 
     database_name = factory.Sequence(lambda _n: "Database %d" % _n)
     database_vendor = SupportedDBVendors.SQLITE_MEM
+    data_source_type = factory.SubFactory(
+        "tests.imp.sql_data.factories.SQLDataSourceTypeFactory"
+    )
 
     @factory.post_generation
     def extract_metadata(
@@ -32,7 +35,8 @@ def extract_metadata(
             "extract_metadata_count", 5
         )
         extract_metadata: Generator[SQLExtractMetadata, Any, Any] = (
-            SQLExtractMetadataFactory() for _ in range(extract_metadata_count)
+            SQLExtractMetadataFactory(data_source=obj)
+            for _ in range(extract_metadata_count)
         )
         obj.extract_metadata = {  # noqa
             _extract_meta.id: _extract_meta
@@ -58,7 +62,8 @@ def data_sources(
     ) -> None:
         data_sources_count: int = kwargs.setdefault("data_sources_count", 5)
         data_sources: Generator[SQLDataSource, Any, Any] = (
-            SQLDataSourceFactory() for _ in range(data_sources_count)
+            SQLDataSourceFactory(data_source_type=obj)
+            for _ in range(data_sources_count)
         )
         obj.data_sources = {  # noqa
             _data_source.id: _data_source for _data_source in data_sources
@@ -76,6 +81,7 @@ class SQLExtractMetadataFactory(ExtractMetadataFactory):
 
     sql_query = "select 'hello world'"
     applicable_source_versions = tuple()
+    data_source = factory.SubFactory(SQLDataSourceFactory)
 
     class Meta:
         model = SQLExtractMetadata

tests/imp/sql_data/test_domain.py

@@ -61,6 +61,7 @@ def test_accessors(self) -> None:
         )
         # This should remain true until after `self.connect_to_db()` is called.
         assert self._data_source.is_disposed
+        assert self._data_source.data_source_type is not None
 
     def test_connect_to_db(self) -> None:
         """Assert that the ``connect_to_db()`` method works as expected."""
@@ -171,6 +172,7 @@ def test_object_initialization_from_a_mapping(self) -> None:
             "description": "Bla bla bla",
             "database_name": "test_db",
             "database_vendor": SupportedDBVendors.SQLITE_MEM,
+            "data_source_type": SQLDataSourceTypeFactory(),
         }
         data_source: SQLDataSource = SQLDataSource.of_mapping(mapping)
 
@@ -236,3 +238,4 @@ def test_accessors(self) -> None:
 
         task = self._extract_meta.to_task()
         assert task is not None
+        assert self._extract_meta.data_source is not None