Merge pull request #120 from zenml-io/michael/docstrings

Add remaining missing docstrings

src/zenml/artifacts/base_artifact.py

@@ -11,8 +11,6 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
-from typing import Any
-
 from tfx.types import Artifact
 
 from zenml.materializers.materializer_factory import MaterializerFactory
@@ -37,13 +35,15 @@ class BaseArtifact(Artifact):
 
     @property
     def materializers(self) -> MaterializerFactory:
-        """Returns a MaterializerFactory which provides access to all registered materializers."""
+        """Returns a MaterializerFactory which provides access
+        to all registered materializers."""
         return MaterializerFactory(self)
 
     @materializers.setter
-    def materializers(self, _: Any):
-        """Setting the materializers property is not allowed. This method always raises
-        an ArtifactInterfaceError with an explanation how to use materializers.
+    def materializers(self, materializers: MaterializerFactory):
+        """Setting the materializers property is not allowed.
+        This method always raises an ArtifactInterfaceError
+        with an explanation how to use materializers.
         """
         raise ArtifactInterfaceError(
             "Setting the materializers property on an artifact is not allowed. "

src/zenml/core/utils.py

@@ -71,6 +71,7 @@ def customise_sources(
         env_settings,
         file_secret_settings,
     ):
+        """Defines precedence of sources to read/write settings from."""
         return (
             init_settings,
             env_settings,

src/zenml/io/gcs_plugin.py

@@ -29,61 +29,131 @@
 
 
 class ZenGCS(filesystem.Filesystem):
+    """Filesystem that delegates to Google Cloud Store using gcsfs."""
+
     SUPPORTED_SCHEMES = ["gs://"]
 
     fs = gcsfs.GCSFileSystem()
 
     @staticmethod
-    def open(name: PathType, mode: str = "r") -> Any:
-        return ZenGCS.fs.open(path=name, mode=mode)
+    def open(path: PathType, mode: str = "r") -> Any:
+        """Open a file at the given path.
+
+        Args:
+            path: Path of the file to open.
+            mode: Mode in which to open the file. Currently only
+                'rb' and 'wb' to read and write binary files are supported.
+        """
+        return ZenGCS.fs.open(path=path, mode=mode)
 
     @staticmethod
     def copy(src: PathType, dst: PathType, overwrite: bool = False) -> None:
-        # TODO: additional params
+        """Copy a file.
+
+        Args:
+            src: The path to copy from.
+            dst: The path to copy to.
+            overwrite: If a file already exists at the destination, this
+                method will overwrite it if overwrite=`True` and
+                raise a FileExistsError otherwise.
+        Raises:
+            FileNotFoundError: If the source file does not exist.
+            FileExistsError: If a file already exists at the destination
+                and overwrite is not set to `True`.
+        """
+        if not overwrite and ZenGCS.fs.exists(dst):
+            raise FileExistsError(
+                f"Unable to copy to destination '{dst}', "
+                f"file already exists. Set `overwrite=True`"
+                f"to copy anyway."
+            )
+
+        # TODO: [LOW] check if it works with overwrite=True or if we need to manually
+        #  remove it first
         ZenGCS.fs.copy(path1=src, path2=dst)
 
     @staticmethod
     def exists(path: PathType) -> bool:
+        """Check whether a path exists."""
         return ZenGCS.fs.exists(path=path)
 
     @staticmethod
     def glob(pattern: PathType) -> List[PathType]:
+        """Return all paths that match the given glob pattern.
+
+        Args:
+            pattern: The glob pattern to match, which may include
+                - '*' to match any number of characters
+                - '?' to match a single character
+                - '[...]' to match one of the characters inside the brackets
+                - '**' as the full name of a path component to match to search
+                  in subdirectories of any depth (e.g. '/some_dir/**/some_file)
+        """
         return ZenGCS.fs.glob(path=pattern)
 
     @staticmethod
     def isdir(path: PathType) -> bool:
+        """Check whether a path is a directory."""
         return ZenGCS.fs.isdir(path=path)
 
     @staticmethod
     def listdir(path: PathType) -> List[PathType]:
+        """Return a list of files in a directory."""
         return ZenGCS.fs.listdir(path=path)
 
     @staticmethod
     def makedirs(path: PathType) -> None:
+        """Create a directory at the given path. If needed also
+        create missing parent directories."""
         ZenGCS.fs.makedirs(path=path, exist_ok=True)
 
     @staticmethod
     def mkdir(path: PathType) -> None:
+        """Create a directory at the given path."""
         ZenGCS.fs.makedir(path=path)
 
     @staticmethod
     def remove(path: PathType) -> None:
+        """Remove the file at the given path."""
         ZenGCS.fs.rm_file(path=path)
 
     @staticmethod
     def rename(src: PathType, dst: PathType, overwrite: bool = False) -> None:
-        # TODO: additional params
+        """Rename source file to destination file.
+
+        Args:
+            src: The path of the file to rename.
+            dst: The path to rename the source file to.
+            overwrite: If a file already exists at the destination, this
+                method will overwrite it if overwrite=`True` and
+                raise a FileExistsError otherwise.
+        Raises:
+            FileNotFoundError: If the source file does not exist.
+            FileExistsError: If a file already exists at the destination
+                and overwrite is not set to `True`.
+        """
+        if not overwrite and ZenGCS.fs.exists(dst):
+            raise FileExistsError(
+                f"Unable to rename file to '{dst}', "
+                f"file already exists. Set `overwrite=True`"
+                f"to rename anyway."
+            )
+
+        # TODO: [LOW] check if it works with overwrite=True or if we need
+        #  to manually remove it first
         ZenGCS.fs.rename(path1=src, path2=dst)
 
     @staticmethod
     def rmtree(path: PathType) -> None:
+        """Remove the given directory."""
         try:
             ZenGCS.fs.delete(path=path, recursive=True)
         except FileNotFoundError as e:
             raise filesystem.NotFoundError() from e
 
     @staticmethod
     def stat(path: PathType) -> Any:
+        """Return stat info for the given path."""
         ZenGCS.fs.stat(path=path)
 
     @staticmethod
@@ -92,7 +162,14 @@ def walk(
         topdown: bool = True,
         onerror: Optional[Callable[..., None]] = None,
     ) -> Iterable[Tuple[PathType, List[PathType], List[PathType]]]:
-        # TODO: additional params
+        """Return an iterator that walks the contents of the given directory.
+
+        Args:
+            top: Path of directory to walk.
+            topdown: Unused argument to conform to interface.
+            onerror: Unused argument to conform to interface.
+        """
+        # TODO: [LOW] additional params
         return ZenGCS.fs.walk(path=top)
 
 

src/zenml/pipelines/base_pipeline.py

@@ -1,16 +1,20 @@
 import inspect
 from abc import abstractmethod
+from typing import Dict
 
 from zenml.annotations.artifact_annotations import Input
 from zenml.annotations.step_annotations import Step
 from zenml.core.repo import Repository
+from zenml.stacks.base_stack import BaseStack
 from zenml.utils.exceptions import PipelineInterfaceError
 
 
 class BasePipelineMeta(type):
-    """ """
+    """Pipeline Metaclass responsible for validating the pipeline definition."""
 
     def __new__(mcs, name, bases, dct):
+        """Ensures that all function arguments are either a `Step`
+        or an `Input`."""
         cls = super().__new__(mcs, name, bases, dct)
         cls.NAME = name
         cls.STEP_SPEC = dict()
@@ -67,53 +71,66 @@ def __init__(self, *args, **kwargs):
 
     @abstractmethod
     def connect(self, *args, **kwargs):
-        """ """
+        """Function that connects inputs and outputs of the pipeline steps."""
 
     @classmethod
     def get_executable(cls):
-        """ """
+        """Returns the `connect` function."""
         return cls.connect
 
     @property
-    def name(self):
+    def name(self) -> str:
         """Name of pipeline is always equal to self.NAME"""
         return self.NAME
 
     @property
-    def stack(self):
+    def stack(self) -> BaseStack:
+        """Returns the stack for this pipeline."""
         return self.__stack
 
     @stack.setter
-    def stack(self, stack):
+    def stack(self, stack: BaseStack):
+        """Setting the stack property is not allowed. This method always
+        raises a PipelineInterfaceError.
+        """
         raise PipelineInterfaceError(
             "The provider will be automatically"
             "inferred from your environment. Please "
             "do no attempt to manually change it."
         )
 
     @property
-    def inputs(self):
+    def inputs(self) -> Dict:
+        """Returns a dictionary of pipeline inputs."""
         return self.__inputs
 
     @inputs.setter
-    def inputs(self, inputs):
+    def inputs(self, inputs: Dict):
+        """Setting the inputs property is not allowed. This method always
+        raises a PipelineInterfaceError.
+        """
         raise PipelineInterfaceError(
             "The provider will be automatically"
             "inferred from your environment. Please "
             "do no attempt to manually change it."
         )
 
     @property
-    def steps(self):
+    def steps(self) -> Dict:
+        """Returns a dictionary of pipeline steps."""
         return self.__steps
 
     @steps.setter
-    def steps(self, steps):
+    def steps(self, steps: Dict):
+        """Setting the steps property is not allowed. This method always
+        raises a PipelineInterfaceError.
+        """
         raise PipelineInterfaceError(
             "The provider will be automatically"
             "inferred from your environment. Please "
             "do no attempt to manually change it."
         )
 
     def run(self):
+        """Runs the pipeline using the orchestrator of the pipeline stack."""
         return self.stack.orchestrator.run(self)