Making utils more consistent

docs/book/developer-guide/repo-and-config.md

@@ -27,7 +27,7 @@ Config Directory_ used on a system can be retrieved by running the following
 command:
 
 ```shell
-python -c 'from zenml.io.utils import get_global_config_directory; print(get_global_config_directory())'
+python -c 'from zenml.utils.io_utils import get_global_config_directory; print(get_global_config_directory())'
 ```
 
 {% hint style="warning" %}

src/zenml/cli/base.py

@@ -29,14 +29,14 @@
 from zenml.constants import REPOSITORY_DIRECTORY_NAME
 from zenml.exceptions import GitNotFoundError, InitializationException
 from zenml.io import fileio
-from zenml.io.utils import copy_dir, get_global_config_directory
 from zenml.logger import get_logger
 from zenml.repository import Repository
 from zenml.utils.analytics_utils import (
     AnalyticsEvent,
     identify_user,
     track_event,
 )
+from zenml.utils.io_utils import copy_dir, get_global_config_directory
 
 logger = get_logger(__name__)
 # WT_SESSION is a Windows Terminal specific environment variable. If it

src/zenml/cli/example.py

@@ -25,7 +25,6 @@
 from rich.markdown import Markdown
 from rich.text import Text
 
-import zenml.io.utils
 from zenml import __version__ as zenml_version_installed
 from zenml.cli.cli import TagGroup, cli
 from zenml.cli.utils import confirmation, declare, error, print_table, warning
@@ -34,6 +33,7 @@
 from zenml.exceptions import GitNotFoundError
 from zenml.io import fileio
 from zenml.logger import get_logger
+from zenml.utils import io_utils
 from zenml.utils.analytics_utils import AnalyticsEvent, track_event
 
 logger = get_logger(__name__)
@@ -64,7 +64,7 @@ def python_files_in_dir(self) -> List[str]:
 
         The `__init__.py` file is excluded from this list.
         """
-        py_in_dir = zenml.io.utils.find_files(str(self.path), "*.py")
+        py_in_dir = io_utils.find_files(str(self.path), "*.py")
         py_files = []
         for file in py_in_dir:
             # Make sure only files directly in dir are considered, not files
@@ -107,9 +107,7 @@ def needs_manual_user_setup(self) -> bool:
         Returns:
             True if no setup.sh file in self.path, False else
         """
-        return not zenml.io.fileio.exists(
-            os.path.join(str(self.path), "setup.sh")
-        )
+        return not fileio.exists(os.path.join(str(self.path), "setup.sh"))
 
     @property
     def executable_python_example(self) -> str:
@@ -358,7 +356,7 @@ class GitExamplesHandler(object):
 
     def __init__(self) -> None:
         """Create a new GitExamplesHandler instance."""
-        self.repo_dir = zenml.io.utils.get_global_config_directory()
+        self.repo_dir = io_utils.get_global_config_directory()
         self.examples_dir = Path(
             os.path.join(self.repo_dir, EXAMPLES_GITHUB_REPO)
         )
@@ -450,8 +448,8 @@ def pull_latest_examples(self) -> None:
 
     def copy_example(self, example: Example, destination_dir: str) -> None:
         """Copies an example to the destination_dir."""
-        zenml.io.utils.create_dir_if_not_exists(destination_dir)
-        zenml.io.utils.copy_dir(
+        io_utils.create_dir_if_not_exists(destination_dir)
+        io_utils.copy_dir(
             str(example.path_in_repo), destination_dir, overwrite=True
         )
 
@@ -638,7 +636,7 @@ def pull(
     git_examples_handler.pull(branch=branch, force=force)
 
     examples_dir = os.path.join(os.getcwd(), path)
-    zenml.io.utils.create_dir_if_not_exists(examples_dir)
+    io_utils.create_dir_if_not_exists(examples_dir)
     try:
         examples = git_examples_handler.get_examples(example_name)
 
@@ -663,7 +661,7 @@ def pull(
 
             declare(f"Pulling example {example.name}...")
 
-            zenml.io.utils.create_dir_if_not_exists(destination_dir)
+            io_utils.create_dir_if_not_exists(destination_dir)
             git_examples_handler.copy_example(example, destination_dir)
             declare(f"Example pulled in directory: {destination_dir}")
             track_event(

src/zenml/cli/server.py

@@ -30,8 +30,8 @@
 from zenml.config.global_config import GlobalConfiguration
 from zenml.console import console
 from zenml.enums import CliCategories
-from zenml.io.utils import get_global_config_directory
 from zenml.logger import get_logger
+from zenml.utils.io_utils import get_global_config_directory
 
 logger = get_logger(__name__)
 GLOBAL_ZENML_SERVER_CONFIG_PATH = os.path.join(

src/zenml/cli/utils.py

@@ -230,14 +230,12 @@ def format_integration_list(
     """
     list_of_dicts = []
     for name, integration_impl in integrations:
-        # type: ignore[attr-defined]
-        is_installed = integration_impl.check_installation()
+        is_installed = integration_impl.check_installation()  # type: ignore[attr-defined]
         list_of_dicts.append(
             {
                 "INSTALLED": ":white_check_mark:" if is_installed else ":x:",
                 "INTEGRATION": name,
-                # type: ignore[attr-defined]
-                "REQUIRED_PACKAGES": ", ".join(integration_impl.REQUIREMENTS),
+                "REQUIRED_PACKAGES": ", ".join(integration_impl.REQUIREMENTS),  # type: ignore[attr-defined]
             }
         )
     return list_of_dicts

src/zenml/config/global_config.py

@@ -27,9 +27,9 @@
     DEFAULT_PROFILE_NAME,
     ProfileConfiguration,
 )
-from zenml.io import fileio, utils
+from zenml.io import fileio
 from zenml.logger import disable_logging, get_logger
-from zenml.utils import yaml_utils
+from zenml.utils import io_utils, yaml_utils
 from zenml.utils.analytics_utils import AnalyticsEvent, track_event
 
 logger = get_logger(__name__)
@@ -294,7 +294,7 @@ def _write_config(self, config_path: Optional[str] = None) -> None:
         logger.debug(f"Writing config to {config_file}")
 
         if not fileio.exists(config_file):
-            utils.create_dir_recursive_if_not_exists(
+            io_utils.create_dir_recursive_if_not_exists(
                 config_path or self.config_directory
             )
 
@@ -303,7 +303,7 @@ def _write_config(self, config_path: Optional[str] = None) -> None:
     @staticmethod
     def default_config_directory() -> str:
         """Path to the default global configuration directory."""
-        return utils.get_global_config_directory()
+        return io_utils.get_global_config_directory()
 
     def _config_file(self, config_path: Optional[str] = None) -> str:
         """Path to the file where global configuration options are stored.

src/zenml/integrations/airflow/orchestrators/airflow_orchestrator.py

@@ -38,14 +38,13 @@
 from pydantic import root_validator
 from tfx.proto.orchestration.pipeline_pb2 import Pipeline as Pb2Pipeline
 
-import zenml.io.utils
 from zenml.integrations.airflow import AIRFLOW_ORCHESTRATOR_FLAVOR
 from zenml.io import fileio
 from zenml.logger import get_logger
 from zenml.orchestrators import BaseOrchestrator
 from zenml.pipelines import Schedule
 from zenml.steps import BaseStep
-from zenml.utils import daemon
+from zenml.utils import daemon, io_utils
 from zenml.utils.source_utils import get_source_root_path
 
 logger = get_logger(__name__)
@@ -184,7 +183,7 @@ def set_airflow_home(cls, values: Dict[str, Any]) -> Dict[str, Any]:
         if "uuid" not in values:
             raise ValueError("`uuid` needs to exist for AirflowOrchestrator.")
         values["airflow_home"] = os.path.join(
-            zenml.io.utils.get_global_config_directory(),
+            io_utils.get_global_config_directory(),
             AIRFLOW_ROOT_DIR,
             str(values["uuid"]),
         )
@@ -226,9 +225,7 @@ def _copy_to_dag_directory_if_necessary(self, dag_filepath: str) -> None:
         Args:
             dag_filepath: Path to the file in which the DAG is defined.
         """
-        dags_directory = zenml.io.utils.resolve_relative_path(
-            self.dags_directory
-        )
+        dags_directory = io_utils.resolve_relative_path(self.dags_directory)
 
         if dags_directory == os.path.dirname(dag_filepath):
             logger.debug("File is already in airflow DAGs directory.")
@@ -343,9 +340,7 @@ def provision(self) -> None:
             return
 
         if not fileio.exists(self.dags_directory):
-            zenml.io.utils.create_dir_recursive_if_not_exists(
-                self.dags_directory
-            )
+            io_utils.create_dir_recursive_if_not_exists(self.dags_directory)
 
         from airflow.cli.commands.standalone_command import StandaloneCommand
 

src/zenml/integrations/azure/artifact_stores/azure_artifact_store.py

@@ -30,7 +30,7 @@
 
 from zenml.artifact_stores import BaseArtifactStore
 from zenml.integrations.azure import AZURE_ARTIFACT_STORE_FLAVOR
-from zenml.io.utils import convert_to_str
+from zenml.utils.io_utils import convert_to_str
 
 PathType = Union[bytes, str]
 

src/zenml/integrations/facets/visualizers/facet_statistics_visualizer.py

@@ -25,10 +25,10 @@
 )
 from IPython.core.display import HTML, display
 
-import zenml.io.utils
 from zenml.environment import Environment
 from zenml.logger import get_logger
 from zenml.post_execution import StepView
+from zenml.utils import io_utils
 from zenml.visualizers import BaseStepVisualizer
 
 logger = get_logger(__name__)
@@ -79,7 +79,7 @@ def generate_html(self, datasets: List[Dict[Text, pd.DataFrame]]) -> str:
             os.path.abspath(os.path.dirname(__file__)),
             "stats.html",
         )
-        html_template = zenml.io.utils.read_file_contents_as_string(template)
+        html_template = io_utils.read_file_contents_as_string(template)
 
         html_ = html_template.replace("protostr", protostr)
         return html_
@@ -99,7 +99,7 @@ def generate_facet(self, html_: str, magic: bool = False) -> None:
             display(HTML(html_))
         else:
             with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f:
-                zenml.io.utils.write_file_contents_as_string(f.name, html_)
+                io_utils.write_file_contents_as_string(f.name, html_)
                 url = f"file:///{f.name}"
                 logger.info("Opening %s in a new browser.." % f.name)
                 webbrowser.open(url, new=2)

src/zenml/integrations/gcp/artifact_stores/gcp_artifact_store.py

@@ -30,7 +30,7 @@
 
 from zenml.artifact_stores import BaseArtifactStore
 from zenml.integrations.gcp import GCP_ARTIFACT_STORE_FLAVOR
-from zenml.io.utils import convert_to_str
+from zenml.utils.io_utils import convert_to_str
 
 PathType = Union[bytes, str]
 

src/zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py

@@ -20,8 +20,8 @@
 from datasets.dataset_dict import DatasetDict
 
 from zenml.artifacts import DataArtifact
-from zenml.io import utils as fileio_utils
 from zenml.materializers.base_materializer import BaseMaterializer
+from zenml.utils import io_utils
 
 DEFAULT_DATASET_DIR = "hf_datasets"
 
@@ -48,6 +48,6 @@ def handle_return(self, ds: Type[Any]) -> None:
         super().handle_return(ds)
         temp_dir = TemporaryDirectory()
         ds.save_to_disk(temp_dir.name)
-        fileio_utils.copy_dir(
+        io_utils.copy_dir(
             temp_dir.name, os.path.join(self.artifact.uri, DEFAULT_DATASET_DIR)
         )

src/zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py

@@ -20,8 +20,8 @@
 from transformers import AutoConfig, PreTrainedModel
 
 from zenml.artifacts import ModelArtifact
-from zenml.io import utils as fileio_utils
 from zenml.materializers.base_materializer import BaseMaterializer
+from zenml.utils import io_utils
 
 DEFAULT_PT_MODEL_DIR = "hf_pt_model"
 
@@ -55,7 +55,7 @@ def handle_return(self, model: Type[Any]) -> None:
         super().handle_return(model)
         temp_dir = TemporaryDirectory()
         model.save_pretrained(temp_dir.name)
-        fileio_utils.copy_dir(
+        io_utils.copy_dir(
             temp_dir.name,
             os.path.join(self.artifact.uri, DEFAULT_PT_MODEL_DIR),
         )

src/zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py

@@ -20,8 +20,8 @@
 from transformers import AutoConfig, TFPreTrainedModel
 
 from zenml.artifacts import ModelArtifact
-from zenml.io import utils as fileio_utils
 from zenml.materializers.base_materializer import BaseMaterializer
+from zenml.utils import io_utils
 
 DEFAULT_TF_MODEL_DIR = "hf_tf_model"
 
@@ -55,7 +55,7 @@ def handle_return(self, model: Type[Any]) -> None:
         super().handle_return(model)
         temp_dir = TemporaryDirectory()
         model.save_pretrained(temp_dir.name)
-        fileio_utils.copy_dir(
+        io_utils.copy_dir(
             temp_dir.name,
             os.path.join(self.artifact.uri, DEFAULT_TF_MODEL_DIR),
         )

src/zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py

@@ -20,8 +20,8 @@
 from transformers.tokenization_utils_base import PreTrainedTokenizerBase
 
 from zenml.artifacts import ModelArtifact
-from zenml.io import utils as fileio_utils
 from zenml.materializers.base_materializer import BaseMaterializer
+from zenml.utils import io_utils
 
 DEFAULT_TOKENIZER_DIR = "hf_tokenizer"
 
@@ -48,7 +48,7 @@ def handle_return(self, tokenizer: Type[Any]) -> None:
         super().handle_return(tokenizer)
         temp_dir = TemporaryDirectory()
         tokenizer.save_pretrained(temp_dir.name)
-        fileio_utils.copy_dir(
+        io_utils.copy_dir(
             temp_dir.name,
             os.path.join(self.artifact.uri, DEFAULT_TOKENIZER_DIR),
         )

src/zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py

@@ -45,7 +45,6 @@
 from pydantic import root_validator
 from tfx.proto.orchestration.pipeline_pb2 import Pipeline as Pb2Pipeline
 
-import zenml.io.utils
 from zenml.artifact_stores import LocalArtifactStore
 from zenml.enums import StackComponentType
 from zenml.environment import Environment
@@ -63,12 +62,11 @@
     KFP_VERSION,
 )
 from zenml.io import fileio
-from zenml.io.utils import get_global_config_directory
 from zenml.logger import get_logger
 from zenml.orchestrators import BaseOrchestrator
 from zenml.repository import Repository
 from zenml.stack import StackValidator
-from zenml.utils import networking_utils
+from zenml.utils import io_utils, networking_utils
 from zenml.utils.docker_utils import get_image_digest
 from zenml.utils.source_utils import get_source_root_path
 
@@ -341,7 +339,7 @@ def root_directory(self) -> str:
         """Returns path to the root directory for all files concerning
         this orchestrator."""
         return os.path.join(
-            zenml.io.utils.get_global_config_directory(),
+            io_utils.get_global_config_directory(),
             "kubeflow",
             str(self.uuid),
         )
@@ -416,7 +414,7 @@ def _configure_container_op(container_op: dsl.ContainerOp) -> None:
         }
 
         stack = Repository().active_stack
-        global_cfg_dir = get_global_config_directory()
+        global_cfg_dir = io_utils.get_global_config_directory()
 
         # go through all stack components and identify those that advertise
         # a local path where they persist information that they need to be
@@ -773,7 +771,7 @@ def list_manual_setup_steps(
             )
             return
 
-        global_config_dir_path = zenml.io.utils.get_global_config_directory()
+        global_config_dir_path = io_utils.get_global_config_directory()
         kubeflow_commands = [
             f"> k3d cluster create {self._k3d_cluster_name} --image {local_deployment_utils.K3S_IMAGE_NAME} --registry-create {container_registry_name} --registry-config {container_registry_path} --volume {global_config_dir_path}:{global_config_dir_path}\n",
             f"> kubectl --context {self.kubernetes_context} apply -k github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref={KFP_VERSION}&timeout=5m",

src/zenml/integrations/kubeflow/orchestrators/local_deployment_utils.py

@@ -4,10 +4,9 @@
 import sys
 import time
 
-import zenml.io.utils
 from zenml.io import fileio
 from zenml.logger import get_logger
-from zenml.utils import networking_utils, yaml_utils
+from zenml.utils import io_utils, networking_utils, yaml_utils
 
 KFP_VERSION = "1.8.1"
 # Name of the K3S image to use for the local K3D cluster.
@@ -94,7 +93,7 @@ def create_k3d_cluster(
         registry_config_path: Path to the registry config file.
     """
     logger.info("Creating local K3D cluster '%s'.", cluster_name)
-    global_config_dir_path = zenml.io.utils.get_global_config_directory()
+    global_config_dir_path = io_utils.get_global_config_directory()
     subprocess.check_call(
         [
             "k3d",