New core architecture

.github/workflows/main.yml

@@ -67,6 +67,7 @@ jobs:
           python -m poetry run zenml integration install pytorch -f
           python -m poetry run zenml integration install mlflow -f
           python -m poetry run zenml integration install gcp -f
+          python -m poetry run zenml integration install kubeflow -f
           python -m poetry run pip install click~=8.0.3
 
       - name: Lint

.github/workflows/pull_request.yml

@@ -66,6 +66,7 @@ jobs:
           python -m poetry run zenml integration install pytorch -f
           python -m poetry run zenml integration install mlflow -f
           python -m poetry run zenml integration install gcp -f
+          python -m poetry run zenml integration install kubeflow -f
           python -m poetry run pip install click~=8.0.3
 
       - name: Lint

docs/book/guides/class-based-api/create-a-step.md

@@ -98,7 +98,7 @@ Step `PandasDatasource` has finished in 0.016s.
 You can add the following code to fetch the pipeline:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 p = repo.get_pipeline(pipeline_name="Chapter1Pipeline")

docs/book/guides/class-based-api/split-and-preprocess.md

@@ -233,7 +233,7 @@ Step `SklearnStandardScaler` has finished in 0.151s.
 You can add the following code to fetch the pipeline:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 p = repo.get_pipeline(pipeline_name="Chapter2Pipeline")

docs/book/guides/class-based-api/train-and-evaluate.md

@@ -228,7 +228,7 @@ Step `SklearnEvaluator` has finished in 0.289s.
 If you add the following code to fetch the pipeline:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 p = repo.get_pipeline(pipeline_name="mnist_pipeline")

docs/book/guides/functional-api/caching.md

@@ -104,7 +104,7 @@ straight to the new trainer and evaluator.
 If you add the following code to fetch the pipeline:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 p = repo.get_pipeline(pipeline_name="mnist_pipeline")

docs/book/guides/functional-api/create-a-step.md

@@ -77,7 +77,7 @@ Step `importer_mnist` has finished in 1.726s.
 You can add the following code to fetch the pipeline:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 p = repo.get_pipeline(pipeline_name="load_mnist_pipeline")

docs/book/guides/functional-api/import-dynamic-data.md

@@ -88,7 +88,7 @@ Even if our data originally lives in an external API, we have now downloaded it
 this pipeline. So we can fetch it and inspect it:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 p = repo.get_pipeline(pipeline_name="mnist_pipeline")

docs/book/guides/functional-api/normalize-data.md

@@ -64,7 +64,7 @@ Step `normalize_mnist` has finished in 1.848s.
 You can add the following code to fetch the pipeline:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 p = repo.get_pipeline(pipeline_name="load_and_normalize_pipeline")

docs/book/guides/functional-api/train-and-evaluate.md

@@ -151,7 +151,7 @@ Step `tf_evaluator` has started.
 If you add the following code to fetch the pipeline:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 p = repo.get_pipeline(pipeline_name="mnist_pipeline")

docs/book/guides/post-execution-workflow/naming-a-pipeline-run.md

@@ -19,7 +19,7 @@ times.
 Once the pipeline run is finished we can easily access this specific run during our post-execution workflow:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 pipeline = repo.get_pipeline(pipeline_name="my_pipeline")

docs/book/guides/post-execution-workflow/post-execution-workflow.md

@@ -20,7 +20,7 @@ repository -> pipelines -> runs -> steps -> outputs
 The highest level `repository` object is where to start from.
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 
 repo = Repository()
 ```

examples/class_based_api/chapter_1.py

@@ -18,9 +18,9 @@
 
 import pandas as pd
 
-from zenml.core.repo import Repository
 from zenml.logger import get_logger
 from zenml.pipelines import BasePipeline
+from zenml.repository import Repository
 from zenml.steps.step_interfaces.base_datasource_step import (
     BaseDatasourceConfig,
     BaseDatasourceStep,

examples/class_based_api/chapter_2.py

@@ -14,10 +14,10 @@
 import os
 from urllib.request import urlopen
 
-from zenml.core.repo import Repository
 from zenml.integrations.sklearn import steps as sklearn_steps
 from zenml.logger import get_logger
 from zenml.pipelines import BasePipeline
+from zenml.repository import Repository
 from zenml.steps import builtin_steps, step_interfaces
 
 logger = get_logger(__name__)

examples/class_based_api/chapter_3.py

@@ -15,11 +15,11 @@
 import os
 from urllib.request import urlopen
 
-from zenml.core.repo import Repository
 from zenml.integrations.sklearn import steps as sklearn_steps
 from zenml.integrations.tensorflow import steps as tf_steps
 from zenml.logger import get_logger
 from zenml.pipelines.builtin_pipelines import TrainingPipeline
+from zenml.repository import Repository
 from zenml.steps import builtin_steps
 
 logger = get_logger(__name__)

examples/dag_visualizer/README.md

@@ -14,7 +14,7 @@ the post-execution workflow we then plug in the visualization class that visuali
 This visualization is produced with the following code:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 from zenml.integrations.graphviz.visualizers.pipeline_run_dag_visualizer import (
     PipelineRunDagVisualizer,
 )

examples/dag_visualizer/run.py

@@ -16,11 +16,11 @@
 import pandas as pd
 import tensorflow as tf
 
-from zenml.core.repo import Repository
 from zenml.integrations.graphviz.visualizers.pipeline_run_dag_visualizer import (
     PipelineRunDagVisualizer,
 )
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import Output, step
 
 FEATURE_COLS = [

examples/drift_detection/evidently.ipynb

@@ -392,7 +392,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zenml.core.repo import Repository\n",
+    "from zenml.repository import Repository\n",
     "from zenml.integrations.evidently.visualizers import EvidentlyVisualizer\n",
     "\n",
     "repo = Repository()\n",

examples/drift_detection/run.py

@@ -12,18 +12,19 @@
 #  permissions and limitations under the License.
 
 import json
+
 import pandas as pd
 from rich import print
 from sklearn import datasets
 
-from zenml.core.repo import Repository
 from zenml.integrations.evidently.steps import (
     EvidentlyProfileConfig,
     EvidentlyProfileStep,
 )
 from zenml.integrations.evidently.visualizers import EvidentlyVisualizer
 from zenml.logger import get_logger
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import step
 
 logger = get_logger(__name__)
@@ -112,14 +113,10 @@ def visualize_statistics():
     repo = Repository()
     pipeline = repo.get_pipelines()[0]
     last_run = pipeline.runs[-1]
-    drift_analysis_step = last_run.get_step(
-        name="drift_analyzer"
-    )
-    print(f'Data drift detected: {drift_analysis_step.output.read()}')
+    drift_analysis_step = last_run.get_step(name="drift_analyzer")
+    print(f"Data drift detected: {drift_analysis_step.output.read()}")
 
-    drift_detection_step = last_run.get_step(
-        name="drift_detector"
-    )
-    print(json.dumps(drift_detection_step.outputs['profile'].read(), indent=2))
+    drift_detection_step = last_run.get_step(name="drift_detector")
+    print(json.dumps(drift_detection_step.outputs["profile"].read(), indent=2))
 
     visualize_statistics()

examples/functional_api/chapter_1.py

@@ -15,8 +15,8 @@
 import numpy as np
 import tensorflow as tf
 
-from zenml.core.repo import Repository
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import Output, step
 
 

examples/functional_api/chapter_2.py

@@ -15,8 +15,8 @@
 import numpy as np
 import tensorflow as tf
 
-from zenml.core.repo import Repository
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import Output, step
 
 

examples/functional_api/chapter_3.py

@@ -15,8 +15,8 @@
 import numpy as np
 import tensorflow as tf
 
-from zenml.core.repo import Repository
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import BaseStepConfig, Output, step
 
 

examples/functional_api/chapter_4.py

@@ -17,8 +17,8 @@
 from sklearn.base import ClassifierMixin
 from sklearn.linear_model import LogisticRegression
 
-from zenml.core.repo import Repository
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import BaseStepConfig, Output, step
 
 

examples/functional_api/chapter_5.py

@@ -23,9 +23,9 @@
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 
-from zenml.core.repo import Repository
 from zenml.materializers.base_materializer import BaseMaterializer
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import BaseStepConfig, Output, step
 
 Base = declarative_base()

examples/functional_api/chapter_6.py

@@ -18,8 +18,8 @@
 from sklearn.base import ClassifierMixin
 from sklearn.linear_model import LogisticRegression
 
-from zenml.core.repo import Repository
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import BaseStepConfig, Output, step
 
 

examples/kubeflow/ZenML Walkthrough Kubeflow Pipelines.ipynb

@@ -603,7 +603,7 @@
    },
    "outputs": [],
    "source": [
-    "from zenml.core.repo import Repository"
+    "from zenml.repository import Repository"
    ]
   },
   {

examples/lineage/README.md

@@ -15,7 +15,7 @@ dataframes for us.
 This visualization is produced with the following code:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 from zenml.integrations.dash.visualizers.pipeline_run_lineage_visualizer import (
     PipelineRunLineageVisualizer,
 )

examples/lineage/run.py

@@ -16,11 +16,11 @@
 import pandas as pd
 import tensorflow as tf
 
-from zenml.core.repo import Repository
 from zenml.integrations.dash.visualizers.pipeline_run_lineage_visualizer import (
     PipelineRunLineageVisualizer,
 )
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import Output, step
 
 FEATURE_COLS = [

examples/not_so_quickstart/run.py

@@ -20,8 +20,8 @@
 from steps.tf_steps import tf_evaluator, tf_trainer
 from steps.torch_steps import torch_evaluator, torch_trainer
 
-from zenml.core.repo import Repository
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import Output, step
 
 

examples/quickstart/quickstart.ipynb

@@ -452,7 +452,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zenml.core.repo import Repository\n",
+    "from zenml.repository import Repository\n",
     "\n",
     "repo = Repository()"
    ]

examples/statistics/README.md

@@ -17,7 +17,7 @@ dataframes for us.
 This visualization is produced with the following code:
 
 ```python
-from zenml.core.repo import Repository
+from zenml.repository import Repository
 from zenml.integrations.facets.visualizers.facet_statistics_visualizer import (
     FacetStatisticsVisualizer,
 )

examples/statistics/run.py

@@ -16,11 +16,11 @@
 import pandas as pd
 import tensorflow as tf
 
-from zenml.core.repo import Repository
 from zenml.integrations.facets.visualizers.facet_statistics_visualizer import (
     FacetStatisticsVisualizer,
 )
 from zenml.pipelines import pipeline
+from zenml.repository import Repository
 from zenml.steps import Output, step
 
 FEATURE_COLS = [

pyproject.toml

@@ -56,7 +56,7 @@ pyyaml = "^5.4.1"
 python-dateutil = "^2.8.1"
 gitpython = "^3.1.18"
 click = "^8.0.1"
-pydantic = "<=1.8.2"
+pydantic = "^1.9.0"
 analytics-python = "^1.4.0"
 distro = "^1.6.0"
 tabulate = "^0.8.9"