diff --git a/docs/conf.py b/docs/conf.py index 4a96a83..d2f9cf8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,6 +59,7 @@ ("py:class", "_RDT"), # private type annotations ("py:class", "_PDT"), # private type annotations ("py:class", "Executor"), # sphinx can't find it + ("py:class", "Future"), # sphinx can't find it ("py:class", "Processor"), # docs aren't published yet ("py:class", "Retry"), # docs aren't published yet ("py:class", "Sink"), # docs aren't published yet diff --git a/docs/index.rst b/docs/index.rst index bc2dbec..668d1fe 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,7 +36,11 @@ API Reference :caption: API :recursive: - sghi.etl.commons + sghi.etl.commons.processors + sghi.etl.commons.sinks + sghi.etl.commons.sources + sghi.etl.commons.utils + sghi.etl.commons.workflow_definitions .. _sghi-etl-core: https://github.com/savannahghi/sghi-etl-core/ diff --git a/src/sghi/etl/commons/sources.py b/src/sghi/etl/commons/sources.py index 85a94b1..7ca0c14 100644 --- a/src/sghi/etl/commons/sources.py +++ b/src/sghi/etl/commons/sources.py @@ -230,11 +230,18 @@ def is_disposed(self) -> bool: def draw(self) -> Sequence[_RDT]: """Draw data from embedded sources and return the aggregated results. + This method retrieves data from all embedded sources asynchronously + using the configured :class:`Executor`. It then applies the provided + result-gatherer function to the :class:`Future` objects resulting from + the asynchronous execution. Each of these ``Future`` objects wraps the + result of drawing from each embedded source contained by this + ``GatherSource``, and they have the same order. + The contents of the resulting aggregate, and their ordering, are - determined by the result-gatherer function provided at this object's - instantiation. That is, the contents of the returned ``Sequence``, will - be the same as those returned by this object's result-gatherer function - and in the same order. + determined by the provided result-gatherer function. That is, the + contents of the returned ``Sequence``, will be the same as those + returned by this object's result-gatherer function and in the same + order. .. admonition:: Don't use after dispose :class: error diff --git a/src/sghi/etl/commons/utils/result_gatherers.py b/src/sghi/etl/commons/utils/result_gatherers.py index 0f5506c..ff6ca6f 100644 --- a/src/sghi/etl/commons/utils/result_gatherers.py +++ b/src/sghi/etl/commons/utils/result_gatherers.py @@ -37,7 +37,7 @@ def fail_fast( exc_wrapper_factory: Callable[[str | None], BaseException] | None = None, exc_wrapper_message: str | None = None, ) -> Iterable[_T]: - """Return the results from futures or errors if any of the futures failed. + """Return the results from futures or error if any of the futures failed. This function returns an ``Iterable`` of the results or errors(if any of the futures failed) gathered from the provided