From bd379689a589a0711a2c73d4a142608e82ed8bb8 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 9 Feb 2022 13:47:51 -0800 Subject: [PATCH 01/39] chore(sdk): move v1 to deprecated --- sdk/python/kfp/{ => deprecated}/__init__.py | 0 sdk/python/kfp/{ => deprecated}/__main__.py | 0 sdk/python/kfp/{ => deprecated}/_auth.py | 0 sdk/python/kfp/{ => deprecated}/_client.py | 10 +++---- sdk/python/kfp/{ => deprecated}/_config.py | 0 .../kfp/{ => deprecated}/_local_client.py | 0 sdk/python/kfp/{ => deprecated}/_runners.py | 0 .../kfp/{ => deprecated}/auth/__init__.py | 0 .../auth/_satvolumecredentials.py | 2 +- .../auth/_tokencredentialsbase.py | 0 sdk/python/kfp/{ => deprecated}/aws.py | 0 sdk/python/kfp/{ => deprecated}/azure.py | 0 .../kfp/{ => deprecated}/cli/__init__.py | 0 sdk/python/kfp/{ => deprecated}/cli/cli.py | 16 +++++------ .../kfp/{ => deprecated}/cli/components.py | 2 +- .../{ => deprecated}/cli/components_test.py | 0 .../cli/diagnose_me/__init__.py | 0 .../cli/diagnose_me/dev_env.py | 2 +- .../cli/diagnose_me/dev_env_test.py | 0 .../{ => deprecated}/cli/diagnose_me/gcp.py | 2 +- .../cli/diagnose_me/gcp_test.py | 4 +-- .../cli/diagnose_me/kubernetes_cluster.py | 2 +- .../diagnose_me/kubernetes_cluster_test.py | 4 +-- .../cli/diagnose_me/utility.py | 0 .../cli/diagnose_me/utility_test.py | 2 +- .../{ => deprecated}/cli/diagnose_me_cli.py | 8 +++--- .../kfp/{ => deprecated}/cli/experiment.py | 2 +- sdk/python/kfp/{ => deprecated}/cli/output.py | 0 .../kfp/{ => deprecated}/cli/pipeline.py | 2 +- .../kfp/{ => deprecated}/cli/recurring_run.py | 2 +- sdk/python/kfp/{ => deprecated}/cli/run.py | 4 +-- .../kfp/{ => deprecated}/compiler/__init__.py | 0 .../compiler/_data_passing_rewriter.py | 4 +-- .../compiler/_data_passing_using_volume.py | 0 .../compiler/_default_transformers.py | 2 +- .../{ => deprecated}/compiler/_k8s_helper.py | 2 +- .../compiler/_op_to_template.py | 6 ++-- .../kfp/{ => deprecated}/compiler/compiler.py | 26 +++++++++--------- .../kfp/{ => deprecated}/compiler/main.py | 4 +-- .../v2_compatible_two_step_pipeline.yaml | 0 ...wo_step_pipeline_with_custom_launcher.yaml | 0 .../{ => deprecated}/compiler/v2_compat.py | 4 +-- .../compiler/v2_compatible_compiler_test.py | 4 +-- .../{ => deprecated}/components/__init__.py | 0 .../components/_airflow_op.py | 0 .../components/_component_store.py | 0 .../components/_components.py | 0 .../components/_data_passing.py | 2 +- .../{ => deprecated}/components/_dynamic.py | 0 .../components/_key_value_store.py | 0 .../{ => deprecated}/components/_naming.py | 0 .../{ => deprecated}/components/_python_op.py | 2 +- .../components/_python_to_graph_component.py | 0 .../components/_structures.py | 0 .../components/_yaml_utils.py | 0 .../{ => deprecated}/components/modelbase.py | 0 .../components/structures/__init__.py | 0 .../structures/components.json_schema.json | 0 .../components.json_schema.outline.yaml | 0 .../components/structures/components.proto | 0 .../structures/generate_proto_code.sh | 0 .../components/type_annotation_utils.py | 0 .../components/type_annotation_utils_test.py | 2 +- .../components_tests/__init__.py | 0 .../components_tests/test_components.py | 8 +++--- ...with_0_inputs_and_2_outputs.component.yaml | 0 ...with_2_inputs_and_0_outputs.component.yaml | 0 ...with_2_inputs_and_2_outputs.component.yaml | 0 .../components_tests/test_data/module1.py | 0 .../module2_which_depends_on_module1.py | 0 .../test_data/python_add.component.yaml | 0 .../test_data/python_add.component.zip | Bin ...tockout_prediction_pipeline.component.yaml | 0 ...il_product_stockout_prediction_pipeline.py | 0 .../components_tests/test_data_passing.py | 2 +- .../components_tests/test_graph_components.py | 6 ++-- .../components_tests/test_python_op.py | 12 ++++---- ...test_python_pipeline_to_graph_component.py | 6 ++-- .../test_structure_model_base.py | 2 +- .../{ => deprecated}/containers/__init__.py | 0 .../containers/_build_image_api.py | 0 .../kfp/{ => deprecated}/containers/_cache.py | 0 .../containers/_component_builder.py | 10 +++---- .../containers/_container_builder.py | 0 .../containers/_gcs_helper.py | 0 .../containers/_k8s_job_helper.py | 0 .../{ => deprecated}/containers/entrypoint.py | 6 ++-- .../containers/entrypoint_utils.py | 6 ++-- .../containers_tests/__init__.py | 0 .../component_builder_test.py | 6 ++-- .../containers_tests/test_build_image_api.py | 4 +-- .../containers_tests/testdata/__init__.py | 0 .../testdata/executor_output.json | 0 .../testdata/expected_component.yaml | 0 .../containers_tests/testdata/main.py | 6 ++-- .../testdata/pipeline_source.py | 0 .../containers_tests/testdata/test_source.py | 0 .../kfp/{ => deprecated}/dsl/__init__.py | 0 .../kfp/{ => deprecated}/dsl/_component.py | 2 +- .../{ => deprecated}/dsl/_component_bridge.py | 22 +++++++-------- .../kfp/{ => deprecated}/dsl/_container_op.py | 6 ++-- .../dsl/_container_op_test.py | 4 +-- .../kfp/{ => deprecated}/dsl/_for_loop.py | 4 +-- .../kfp/{ => deprecated}/dsl/_metadata.py | 2 +- .../kfp/{ => deprecated}/dsl/_ops_group.py | 2 +- .../kfp/{ => deprecated}/dsl/_pipeline.py | 12 ++++---- .../{ => deprecated}/dsl/_pipeline_param.py | 0 .../{ => deprecated}/dsl/_pipeline_volume.py | 0 .../kfp/{ => deprecated}/dsl/_resource_op.py | 0 .../kfp/{ => deprecated}/dsl/_volume_op.py | 0 .../dsl/_volume_snapshot_op.py | 0 .../kfp/{ => deprecated}/dsl/artifact.py | 4 +-- .../{ => deprecated}/dsl/artifact_utils.py | 0 .../{ => deprecated}/dsl/component_spec.py | 4 +-- .../dsl/component_spec_test.py | 6 ++-- .../dsl/data_passing_methods.py | 0 .../kfp/{ => deprecated}/dsl/dsl_utils.py | 2 +- .../{ => deprecated}/dsl/dsl_utils_test.py | 2 +- .../dsl/extensions/__init__.py | 0 .../dsl/extensions/kubernetes.py | 0 .../kfp/{ => deprecated}/dsl/io_types.py | 0 .../kfp/{ => deprecated}/dsl/metrics_utils.py | 2 +- .../dsl/metrics_utils_test.py | 2 +- .../dsl/serialization_utils.py | 0 .../dsl/serialization_utils_test.py | 2 +- ...expected_bulk_loaded_confusion_matrix.json | 0 .../test_data/expected_confusion_matrix.json | 0 .../type_schemas/classification_metrics.yaml | 0 .../dsl/type_schemas/confidence_metrics.yaml | 0 .../dsl/type_schemas/confusion_matrix.yaml | 0 .../dsl/type_schemas/dataset.yaml | 0 .../dsl/type_schemas/metrics.yaml | 0 .../dsl/type_schemas/model.yaml | 0 .../sliced_classification_metrics.yaml | 0 .../kfp/{ => deprecated}/dsl/type_utils.py | 0 sdk/python/kfp/{ => deprecated}/dsl/types.py | 2 +- sdk/python/kfp/{ => deprecated}/gcp.py | 0 .../kfp/{ => deprecated}/notebook/__init__.py | 0 .../kfp/{ => deprecated}/notebook/_magic.py | 0 sdk/python/kfp/{ => deprecated}/onprem.py | 2 +- sdk/python/kfp/v2/components/structures.py | 4 +-- .../kfp/v2/components/types/type_utils.py | 2 +- 142 files changed, 142 insertions(+), 142 deletions(-) rename sdk/python/kfp/{ => deprecated}/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/__main__.py (100%) rename sdk/python/kfp/{ => deprecated}/_auth.py (100%) rename sdk/python/kfp/{ => deprecated}/_client.py (99%) rename sdk/python/kfp/{ => deprecated}/_config.py (100%) rename sdk/python/kfp/{ => deprecated}/_local_client.py (100%) rename sdk/python/kfp/{ => deprecated}/_runners.py (100%) rename sdk/python/kfp/{ => deprecated}/auth/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/auth/_satvolumecredentials.py (98%) rename sdk/python/kfp/{ => deprecated}/auth/_tokencredentialsbase.py (100%) rename sdk/python/kfp/{ => deprecated}/aws.py (100%) rename sdk/python/kfp/{ => deprecated}/azure.py (100%) rename sdk/python/kfp/{ => deprecated}/cli/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/cli/cli.py (86%) rename sdk/python/kfp/{ => deprecated}/cli/components.py (99%) rename sdk/python/kfp/{ => deprecated}/cli/components_test.py (100%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me/dev_env.py (97%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me/dev_env_test.py (100%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me/gcp.py (99%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me/gcp_test.py (97%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me/kubernetes_cluster.py (98%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me/kubernetes_cluster_test.py (96%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me/utility.py (100%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me/utility_test.py (96%) rename sdk/python/kfp/{ => deprecated}/cli/diagnose_me_cli.py (94%) rename sdk/python/kfp/{ => deprecated}/cli/experiment.py (98%) rename sdk/python/kfp/{ => deprecated}/cli/output.py (100%) rename sdk/python/kfp/{ => deprecated}/cli/pipeline.py (99%) rename sdk/python/kfp/{ => deprecated}/cli/recurring_run.py (99%) rename sdk/python/kfp/{ => deprecated}/cli/run.py (98%) rename sdk/python/kfp/{ => deprecated}/compiler/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/compiler/_data_passing_rewriter.py (99%) rename sdk/python/kfp/{ => deprecated}/compiler/_data_passing_using_volume.py (100%) rename sdk/python/kfp/{ => deprecated}/compiler/_default_transformers.py (98%) rename sdk/python/kfp/{ => deprecated}/compiler/_k8s_helper.py (99%) rename sdk/python/kfp/{ => deprecated}/compiler/_op_to_template.py (98%) rename sdk/python/kfp/{ => deprecated}/compiler/compiler.py (98%) rename sdk/python/kfp/{ => deprecated}/compiler/main.py (98%) rename sdk/python/kfp/{ => deprecated}/compiler/testdata/v2_compatible_two_step_pipeline.yaml (100%) rename sdk/python/kfp/{ => deprecated}/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml (100%) rename sdk/python/kfp/{ => deprecated}/compiler/v2_compat.py (98%) rename sdk/python/kfp/{ => deprecated}/compiler/v2_compatible_compiler_test.py (98%) rename sdk/python/kfp/{ => deprecated}/components/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/components/_airflow_op.py (100%) rename sdk/python/kfp/{ => deprecated}/components/_component_store.py (100%) rename sdk/python/kfp/{ => deprecated}/components/_components.py (100%) rename sdk/python/kfp/{ => deprecated}/components/_data_passing.py (99%) rename sdk/python/kfp/{ => deprecated}/components/_dynamic.py (100%) rename sdk/python/kfp/{ => deprecated}/components/_key_value_store.py (100%) rename sdk/python/kfp/{ => deprecated}/components/_naming.py (100%) rename sdk/python/kfp/{ => deprecated}/components/_python_op.py (99%) rename sdk/python/kfp/{ => deprecated}/components/_python_to_graph_component.py (100%) rename sdk/python/kfp/{ => deprecated}/components/_structures.py (100%) rename sdk/python/kfp/{ => deprecated}/components/_yaml_utils.py (100%) rename sdk/python/kfp/{ => deprecated}/components/modelbase.py (100%) rename sdk/python/kfp/{ => deprecated}/components/structures/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/components/structures/components.json_schema.json (100%) rename sdk/python/kfp/{ => deprecated}/components/structures/components.json_schema.outline.yaml (100%) rename sdk/python/kfp/{ => deprecated}/components/structures/components.proto (100%) rename sdk/python/kfp/{ => deprecated}/components/structures/generate_proto_code.sh (100%) rename sdk/python/kfp/{ => deprecated}/components/type_annotation_utils.py (100%) rename sdk/python/kfp/{ => deprecated}/components/type_annotation_utils_test.py (97%) rename sdk/python/kfp/{ => deprecated}/components_tests/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_components.py (99%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data/component_with_0_inputs_and_2_outputs.component.yaml (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data/component_with_2_inputs_and_0_outputs.component.yaml (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data/component_with_2_inputs_and_2_outputs.component.yaml (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data/module1.py (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data/module2_which_depends_on_module1.py (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data/python_add.component.yaml (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data/python_add.component.zip (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data/retail_product_stockout_prediction_pipeline.component.yaml (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data/retail_product_stockout_prediction_pipeline.py (100%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_data_passing.py (95%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_graph_components.py (96%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_python_op.py (99%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_python_pipeline_to_graph_component.py (91%) rename sdk/python/kfp/{ => deprecated}/components_tests/test_structure_model_base.py (99%) rename sdk/python/kfp/{ => deprecated}/containers/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/containers/_build_image_api.py (100%) rename sdk/python/kfp/{ => deprecated}/containers/_cache.py (100%) rename sdk/python/kfp/{ => deprecated}/containers/_component_builder.py (98%) rename sdk/python/kfp/{ => deprecated}/containers/_container_builder.py (100%) rename sdk/python/kfp/{ => deprecated}/containers/_gcs_helper.py (100%) rename sdk/python/kfp/{ => deprecated}/containers/_k8s_job_helper.py (100%) rename sdk/python/kfp/{ => deprecated}/containers/entrypoint.py (98%) rename sdk/python/kfp/{ => deprecated}/containers/entrypoint_utils.py (98%) rename sdk/python/kfp/{ => deprecated}/containers_tests/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/containers_tests/component_builder_test.py (93%) rename sdk/python/kfp/{ => deprecated}/containers_tests/test_build_image_api.py (98%) rename sdk/python/kfp/{ => deprecated}/containers_tests/testdata/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/containers_tests/testdata/executor_output.json (100%) rename sdk/python/kfp/{ => deprecated}/containers_tests/testdata/expected_component.yaml (100%) rename sdk/python/kfp/{ => deprecated}/containers_tests/testdata/main.py (94%) rename sdk/python/kfp/{ => deprecated}/containers_tests/testdata/pipeline_source.py (100%) rename sdk/python/kfp/{ => deprecated}/containers_tests/testdata/test_source.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/_component.py (99%) rename sdk/python/kfp/{ => deprecated}/dsl/_component_bridge.py (98%) rename sdk/python/kfp/{ => deprecated}/dsl/_container_op.py (99%) rename sdk/python/kfp/{ => deprecated}/dsl/_container_op_test.py (94%) rename sdk/python/kfp/{ => deprecated}/dsl/_for_loop.py (99%) rename sdk/python/kfp/{ => deprecated}/dsl/_metadata.py (97%) rename sdk/python/kfp/{ => deprecated}/dsl/_ops_group.py (99%) rename sdk/python/kfp/{ => deprecated}/dsl/_pipeline.py (97%) rename sdk/python/kfp/{ => deprecated}/dsl/_pipeline_param.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/_pipeline_volume.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/_resource_op.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/_volume_op.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/_volume_snapshot_op.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/artifact.py (98%) rename sdk/python/kfp/{ => deprecated}/dsl/artifact_utils.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/component_spec.py (99%) rename sdk/python/kfp/{ => deprecated}/dsl/component_spec_test.py (99%) rename sdk/python/kfp/{ => deprecated}/dsl/data_passing_methods.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/dsl_utils.py (99%) rename sdk/python/kfp/{ => deprecated}/dsl/dsl_utils_test.py (98%) rename sdk/python/kfp/{ => deprecated}/dsl/extensions/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/extensions/kubernetes.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/io_types.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/metrics_utils.py (99%) rename sdk/python/kfp/{ => deprecated}/dsl/metrics_utils_test.py (98%) rename sdk/python/kfp/{ => deprecated}/dsl/serialization_utils.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/serialization_utils_test.py (95%) rename sdk/python/kfp/{ => deprecated}/dsl/test_data/expected_bulk_loaded_confusion_matrix.json (100%) rename sdk/python/kfp/{ => deprecated}/dsl/test_data/expected_confusion_matrix.json (100%) rename sdk/python/kfp/{ => deprecated}/dsl/type_schemas/classification_metrics.yaml (100%) rename sdk/python/kfp/{ => deprecated}/dsl/type_schemas/confidence_metrics.yaml (100%) rename sdk/python/kfp/{ => deprecated}/dsl/type_schemas/confusion_matrix.yaml (100%) rename sdk/python/kfp/{ => deprecated}/dsl/type_schemas/dataset.yaml (100%) rename sdk/python/kfp/{ => deprecated}/dsl/type_schemas/metrics.yaml (100%) rename sdk/python/kfp/{ => deprecated}/dsl/type_schemas/model.yaml (100%) rename sdk/python/kfp/{ => deprecated}/dsl/type_schemas/sliced_classification_metrics.yaml (100%) rename sdk/python/kfp/{ => deprecated}/dsl/type_utils.py (100%) rename sdk/python/kfp/{ => deprecated}/dsl/types.py (99%) rename sdk/python/kfp/{ => deprecated}/gcp.py (100%) rename sdk/python/kfp/{ => deprecated}/notebook/__init__.py (100%) rename sdk/python/kfp/{ => deprecated}/notebook/_magic.py (100%) rename sdk/python/kfp/{ => deprecated}/onprem.py (99%) diff --git a/sdk/python/kfp/__init__.py b/sdk/python/kfp/deprecated/__init__.py similarity index 100% rename from sdk/python/kfp/__init__.py rename to sdk/python/kfp/deprecated/__init__.py diff --git a/sdk/python/kfp/__main__.py b/sdk/python/kfp/deprecated/__main__.py similarity index 100% rename from sdk/python/kfp/__main__.py rename to sdk/python/kfp/deprecated/__main__.py diff --git a/sdk/python/kfp/_auth.py b/sdk/python/kfp/deprecated/_auth.py similarity index 100% rename from sdk/python/kfp/_auth.py rename to sdk/python/kfp/deprecated/_auth.py diff --git a/sdk/python/kfp/_client.py b/sdk/python/kfp/deprecated/_client.py similarity index 99% rename from sdk/python/kfp/_client.py rename to sdk/python/kfp/deprecated/_client.py index 633252895fc..cd79ce2f691 100644 --- a/sdk/python/kfp/_client.py +++ b/sdk/python/kfp/deprecated/_client.py @@ -28,11 +28,11 @@ import kfp_server_api -from kfp import dsl -from kfp.compiler import compiler -from kfp.compiler._k8s_helper import sanitize_k8s_name +from kfp.deprecated import dsl +from kfp.deprecated.compiler import compiler +from kfp.deprecated.compiler._k8s_helper import sanitize_k8s_name -from kfp._auth import get_auth_token, get_gcp_access_token +from kfp.deprecated._auth import get_auth_token, get_gcp_access_token from kfp_server_api import ApiException # Operators on scalar values. Only applies to one of |int_value|, @@ -1146,7 +1146,7 @@ def disable_job(self, job_id: str): ApiException: If the job is not found. """ return self._job_api.disable_job(id=job_id) - + def enable_job(self, job_id: str): """Enables a job. diff --git a/sdk/python/kfp/_config.py b/sdk/python/kfp/deprecated/_config.py similarity index 100% rename from sdk/python/kfp/_config.py rename to sdk/python/kfp/deprecated/_config.py diff --git a/sdk/python/kfp/_local_client.py b/sdk/python/kfp/deprecated/_local_client.py similarity index 100% rename from sdk/python/kfp/_local_client.py rename to sdk/python/kfp/deprecated/_local_client.py diff --git a/sdk/python/kfp/_runners.py b/sdk/python/kfp/deprecated/_runners.py similarity index 100% rename from sdk/python/kfp/_runners.py rename to sdk/python/kfp/deprecated/_runners.py diff --git a/sdk/python/kfp/auth/__init__.py b/sdk/python/kfp/deprecated/auth/__init__.py similarity index 100% rename from sdk/python/kfp/auth/__init__.py rename to sdk/python/kfp/deprecated/auth/__init__.py diff --git a/sdk/python/kfp/auth/_satvolumecredentials.py b/sdk/python/kfp/deprecated/auth/_satvolumecredentials.py similarity index 98% rename from sdk/python/kfp/auth/_satvolumecredentials.py rename to sdk/python/kfp/deprecated/auth/_satvolumecredentials.py index 660b3b9961b..cb31a342a30 100644 --- a/sdk/python/kfp/auth/_satvolumecredentials.py +++ b/sdk/python/kfp/deprecated/auth/_satvolumecredentials.py @@ -17,7 +17,7 @@ from kubernetes.client import configuration -from kfp import auth +from kfp.deprecated import auth class ServiceAccountTokenVolumeCredentials(auth.TokenCredentialsBase): diff --git a/sdk/python/kfp/auth/_tokencredentialsbase.py b/sdk/python/kfp/deprecated/auth/_tokencredentialsbase.py similarity index 100% rename from sdk/python/kfp/auth/_tokencredentialsbase.py rename to sdk/python/kfp/deprecated/auth/_tokencredentialsbase.py diff --git a/sdk/python/kfp/aws.py b/sdk/python/kfp/deprecated/aws.py similarity index 100% rename from sdk/python/kfp/aws.py rename to sdk/python/kfp/deprecated/aws.py diff --git a/sdk/python/kfp/azure.py b/sdk/python/kfp/deprecated/azure.py similarity index 100% rename from sdk/python/kfp/azure.py rename to sdk/python/kfp/deprecated/azure.py diff --git a/sdk/python/kfp/cli/__init__.py b/sdk/python/kfp/deprecated/cli/__init__.py similarity index 100% rename from sdk/python/kfp/cli/__init__.py rename to sdk/python/kfp/deprecated/cli/__init__.py diff --git a/sdk/python/kfp/cli/cli.py b/sdk/python/kfp/deprecated/cli/cli.py similarity index 86% rename from sdk/python/kfp/cli/cli.py rename to sdk/python/kfp/deprecated/cli/cli.py index fe49cc01d7a..3867e6958ae 100644 --- a/sdk/python/kfp/cli/cli.py +++ b/sdk/python/kfp/deprecated/cli/cli.py @@ -18,14 +18,14 @@ import click import typer -from kfp._client import Client -from kfp.cli.run import run -from kfp.cli.recurring_run import recurring_run -from kfp.cli.pipeline import pipeline -from kfp.cli.diagnose_me_cli import diagnose_me -from kfp.cli.experiment import experiment -from kfp.cli.output import OutputFormat -from kfp.cli import components +from kfp.deprecated._client import Client +from kfp.deprecated.cli.run import run +from kfp.deprecated.cli.recurring_run import recurring_run +from kfp.deprecated.cli.pipeline import pipeline +from kfp.deprecated.cli.diagnose_me_cli import diagnose_me +from kfp.deprecated.cli.experiment import experiment +from kfp.deprecated.cli.output import OutputFormat +from kfp.deprecated.cli import components @click.group() diff --git a/sdk/python/kfp/cli/components.py b/sdk/python/kfp/deprecated/cli/components.py similarity index 99% rename from sdk/python/kfp/cli/components.py rename to sdk/python/kfp/deprecated/cli/components.py index 5785b8dd2a3..fc40235c463 100644 --- a/sdk/python/kfp/cli/components.py +++ b/sdk/python/kfp/deprecated/cli/components.py @@ -28,7 +28,7 @@ import typer -import kfp +import kfp.deprecated from kfp.v2.components import component_factory, kfp_config, utils _REQUIREMENTS_TXT = 'requirements.txt' diff --git a/sdk/python/kfp/cli/components_test.py b/sdk/python/kfp/deprecated/cli/components_test.py similarity index 100% rename from sdk/python/kfp/cli/components_test.py rename to sdk/python/kfp/deprecated/cli/components_test.py diff --git a/sdk/python/kfp/cli/diagnose_me/__init__.py b/sdk/python/kfp/deprecated/cli/diagnose_me/__init__.py similarity index 100% rename from sdk/python/kfp/cli/diagnose_me/__init__.py rename to sdk/python/kfp/deprecated/cli/diagnose_me/__init__.py diff --git a/sdk/python/kfp/cli/diagnose_me/dev_env.py b/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env.py similarity index 97% rename from sdk/python/kfp/cli/diagnose_me/dev_env.py rename to sdk/python/kfp/deprecated/cli/diagnose_me/dev_env.py index 9d6a1d256ce..9a5267590c2 100644 --- a/sdk/python/kfp/cli/diagnose_me/dev_env.py +++ b/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env.py @@ -15,7 +15,7 @@ """Functions for diagnostic data collection from development development.""" import enum -from kfp.cli.diagnose_me import utility +from kfp.deprecated.cli.diagnose_me import utility class Commands(enum.Enum): diff --git a/sdk/python/kfp/cli/diagnose_me/dev_env_test.py b/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py similarity index 100% rename from sdk/python/kfp/cli/diagnose_me/dev_env_test.py rename to sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py diff --git a/sdk/python/kfp/cli/diagnose_me/gcp.py b/sdk/python/kfp/deprecated/cli/diagnose_me/gcp.py similarity index 99% rename from sdk/python/kfp/cli/diagnose_me/gcp.py rename to sdk/python/kfp/deprecated/cli/diagnose_me/gcp.py index 6e43e1ac88c..be643320f81 100644 --- a/sdk/python/kfp/cli/diagnose_me/gcp.py +++ b/sdk/python/kfp/deprecated/cli/diagnose_me/gcp.py @@ -16,7 +16,7 @@ import enum from typing import List, Text, Optional -from kfp.cli.diagnose_me import utility +from kfp.deprecated.cli.diagnose_me import utility class Commands(enum.Enum): diff --git a/sdk/python/kfp/cli/diagnose_me/gcp_test.py b/sdk/python/kfp/deprecated/cli/diagnose_me/gcp_test.py similarity index 97% rename from sdk/python/kfp/cli/diagnose_me/gcp_test.py rename to sdk/python/kfp/deprecated/cli/diagnose_me/gcp_test.py index fd19d874e1c..630708675aa 100644 --- a/sdk/python/kfp/cli/diagnose_me/gcp_test.py +++ b/sdk/python/kfp/deprecated/cli/diagnose_me/gcp_test.py @@ -17,8 +17,8 @@ from typing import Text import unittest from unittest import mock -from kfp.cli.diagnose_me import gcp -from kfp.cli.diagnose_me import utility +from kfp.deprecated.cli.diagnose_me import gcp +from kfp.deprecated.cli.diagnose_me import utility class GoogleCloudTest(unittest.TestCase): diff --git a/sdk/python/kfp/cli/diagnose_me/kubernetes_cluster.py b/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster.py similarity index 98% rename from sdk/python/kfp/cli/diagnose_me/kubernetes_cluster.py rename to sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster.py index cbb26920598..43d44adfdbd 100644 --- a/sdk/python/kfp/cli/diagnose_me/kubernetes_cluster.py +++ b/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster.py @@ -16,7 +16,7 @@ import enum from typing import List, Text -from kfp.cli.diagnose_me import utility +from kfp.deprecated.cli.diagnose_me import utility class Commands(enum.Enum): diff --git a/sdk/python/kfp/cli/diagnose_me/kubernetes_cluster_test.py b/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster_test.py similarity index 96% rename from sdk/python/kfp/cli/diagnose_me/kubernetes_cluster_test.py rename to sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster_test.py index 6aff5c2c957..e2409797b22 100644 --- a/sdk/python/kfp/cli/diagnose_me/kubernetes_cluster_test.py +++ b/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster_test.py @@ -17,8 +17,8 @@ from typing import Text import unittest from unittest import mock -from kfp.cli.diagnose_me import kubernetes_cluster as dkc -from kfp.cli.diagnose_me import utility +from kfp.deprecated.cli.diagnose_me import kubernetes_cluster as dkc +from kfp.deprecated.cli.diagnose_me import utility class KubernetesClusterTest(unittest.TestCase): diff --git a/sdk/python/kfp/cli/diagnose_me/utility.py b/sdk/python/kfp/deprecated/cli/diagnose_me/utility.py similarity index 100% rename from sdk/python/kfp/cli/diagnose_me/utility.py rename to sdk/python/kfp/deprecated/cli/diagnose_me/utility.py diff --git a/sdk/python/kfp/cli/diagnose_me/utility_test.py b/sdk/python/kfp/deprecated/cli/diagnose_me/utility_test.py similarity index 96% rename from sdk/python/kfp/cli/diagnose_me/utility_test.py rename to sdk/python/kfp/deprecated/cli/diagnose_me/utility_test.py index 18810a4575f..f339d711429 100644 --- a/sdk/python/kfp/cli/diagnose_me/utility_test.py +++ b/sdk/python/kfp/deprecated/cli/diagnose_me/utility_test.py @@ -15,7 +15,7 @@ """Tests for diagnose_me.utility.""" import unittest -from kfp.cli.diagnose_me import utility +from kfp.deprecated.cli.diagnose_me import utility class UtilityTest(unittest.TestCase): diff --git a/sdk/python/kfp/cli/diagnose_me_cli.py b/sdk/python/kfp/deprecated/cli/diagnose_me_cli.py similarity index 94% rename from sdk/python/kfp/cli/diagnose_me_cli.py rename to sdk/python/kfp/deprecated/cli/diagnose_me_cli.py index 718a6d76730..c6a5863748b 100644 --- a/sdk/python/kfp/cli/diagnose_me_cli.py +++ b/sdk/python/kfp/deprecated/cli/diagnose_me_cli.py @@ -5,10 +5,10 @@ import sys from typing import Dict, Text import click -from kfp.cli.diagnose_me import dev_env -from kfp.cli.diagnose_me import gcp -from kfp.cli.diagnose_me import kubernetes_cluster as k8 -from kfp.cli.diagnose_me import utility +from kfp.deprecated.cli.diagnose_me import dev_env +from kfp.deprecated.cli.diagnose_me import gcp +from kfp.deprecated.cli.diagnose_me import kubernetes_cluster as k8 +from kfp.deprecated.cli.diagnose_me import utility @click.group() diff --git a/sdk/python/kfp/cli/experiment.py b/sdk/python/kfp/deprecated/cli/experiment.py similarity index 98% rename from sdk/python/kfp/cli/experiment.py rename to sdk/python/kfp/deprecated/cli/experiment.py index a45a0e6baf8..262a5a8ca3f 100644 --- a/sdk/python/kfp/cli/experiment.py +++ b/sdk/python/kfp/deprecated/cli/experiment.py @@ -2,7 +2,7 @@ import json from typing import List -from kfp.cli.output import print_output, OutputFormat +from kfp.deprecated.cli.output import print_output, OutputFormat import kfp_server_api from kfp_server_api.models.api_experiment import ApiExperiment diff --git a/sdk/python/kfp/cli/output.py b/sdk/python/kfp/deprecated/cli/output.py similarity index 100% rename from sdk/python/kfp/cli/output.py rename to sdk/python/kfp/deprecated/cli/output.py diff --git a/sdk/python/kfp/cli/pipeline.py b/sdk/python/kfp/deprecated/cli/pipeline.py similarity index 99% rename from sdk/python/kfp/cli/pipeline.py rename to sdk/python/kfp/deprecated/cli/pipeline.py index add7237b52a..db3838f8f63 100644 --- a/sdk/python/kfp/cli/pipeline.py +++ b/sdk/python/kfp/deprecated/cli/pipeline.py @@ -17,7 +17,7 @@ from typing import List, Optional import kfp_server_api -from kfp.cli.output import print_output, OutputFormat +from kfp.deprecated.cli.output import print_output, OutputFormat @click.group() diff --git a/sdk/python/kfp/cli/recurring_run.py b/sdk/python/kfp/deprecated/cli/recurring_run.py similarity index 99% rename from sdk/python/kfp/cli/recurring_run.py rename to sdk/python/kfp/deprecated/cli/recurring_run.py index 78a3be4eca5..fc0f20b2730 100644 --- a/sdk/python/kfp/cli/recurring_run.py +++ b/sdk/python/kfp/deprecated/cli/recurring_run.py @@ -15,7 +15,7 @@ from typing import Any, Dict, List, Optional import json import click -from kfp.cli.output import print_output, OutputFormat +from kfp.deprecated.cli.output import print_output, OutputFormat import kfp_server_api diff --git a/sdk/python/kfp/cli/run.py b/sdk/python/kfp/deprecated/cli/run.py similarity index 98% rename from sdk/python/kfp/cli/run.py rename to sdk/python/kfp/deprecated/cli/run.py index b8620b882a5..19bd49d901f 100644 --- a/sdk/python/kfp/cli/run.py +++ b/sdk/python/kfp/deprecated/cli/run.py @@ -21,8 +21,8 @@ from typing import List import kfp_server_api -from kfp._client import Client -from kfp.cli.output import print_output, OutputFormat +from kfp.deprecated._client import Client +from kfp.deprecated.cli.output import print_output, OutputFormat @click.group() diff --git a/sdk/python/kfp/compiler/__init__.py b/sdk/python/kfp/deprecated/compiler/__init__.py similarity index 100% rename from sdk/python/kfp/compiler/__init__.py rename to sdk/python/kfp/deprecated/compiler/__init__.py diff --git a/sdk/python/kfp/compiler/_data_passing_rewriter.py b/sdk/python/kfp/deprecated/compiler/_data_passing_rewriter.py similarity index 99% rename from sdk/python/kfp/compiler/_data_passing_rewriter.py rename to sdk/python/kfp/deprecated/compiler/_data_passing_rewriter.py index 64e74f8db56..7292242d6ba 100644 --- a/sdk/python/kfp/compiler/_data_passing_rewriter.py +++ b/sdk/python/kfp/deprecated/compiler/_data_passing_rewriter.py @@ -4,8 +4,8 @@ import re from typing import Any, Dict, List, Optional, Set, Tuple -from kfp.dsl import _component_bridge -from kfp import dsl +from kfp.deprecated.dsl import _component_bridge +from kfp.deprecated import dsl def fix_big_data_passing(workflow: dict) -> dict: diff --git a/sdk/python/kfp/compiler/_data_passing_using_volume.py b/sdk/python/kfp/deprecated/compiler/_data_passing_using_volume.py similarity index 100% rename from sdk/python/kfp/compiler/_data_passing_using_volume.py rename to sdk/python/kfp/deprecated/compiler/_data_passing_using_volume.py diff --git a/sdk/python/kfp/compiler/_default_transformers.py b/sdk/python/kfp/deprecated/compiler/_default_transformers.py similarity index 98% rename from sdk/python/kfp/compiler/_default_transformers.py rename to sdk/python/kfp/deprecated/compiler/_default_transformers.py index 9a88621a2fd..0bf9cf5677f 100644 --- a/sdk/python/kfp/compiler/_default_transformers.py +++ b/sdk/python/kfp/deprecated/compiler/_default_transformers.py @@ -15,7 +15,7 @@ import warnings from kubernetes import client as k8s_client from typing import Callable, Dict, Optional, Text -from kfp.dsl._container_op import BaseOp, ContainerOp +from kfp.deprecated.dsl._container_op import BaseOp, ContainerOp def add_pod_env(op: BaseOp) -> BaseOp: diff --git a/sdk/python/kfp/compiler/_k8s_helper.py b/sdk/python/kfp/deprecated/compiler/_k8s_helper.py similarity index 99% rename from sdk/python/kfp/compiler/_k8s_helper.py rename to sdk/python/kfp/deprecated/compiler/_k8s_helper.py index f11b26ab079..a2f77a59382 100644 --- a/sdk/python/kfp/compiler/_k8s_helper.py +++ b/sdk/python/kfp/deprecated/compiler/_k8s_helper.py @@ -14,7 +14,7 @@ import re -from kfp import dsl +from kfp.deprecated import dsl def sanitize_k8s_name(name, allow_capital_underscore=False): diff --git a/sdk/python/kfp/compiler/_op_to_template.py b/sdk/python/kfp/deprecated/compiler/_op_to_template.py similarity index 98% rename from sdk/python/kfp/compiler/_op_to_template.py rename to sdk/python/kfp/deprecated/compiler/_op_to_template.py index 538b1474e39..db9cef22ba2 100644 --- a/sdk/python/kfp/compiler/_op_to_template.py +++ b/sdk/python/kfp/deprecated/compiler/_op_to_template.py @@ -20,9 +20,9 @@ from collections import OrderedDict from typing import Union, List, Any, Callable, TypeVar, Dict -from kfp.compiler._k8s_helper import convert_k8s_obj_to_json -from kfp import dsl -from kfp.dsl._container_op import BaseOp +from kfp.deprecated.compiler._k8s_helper import convert_k8s_obj_to_json +from kfp.deprecated import dsl +from kfp.deprecated.dsl._container_op import BaseOp # generics T = TypeVar('T') diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/deprecated/compiler/compiler.py similarity index 98% rename from sdk/python/kfp/compiler/compiler.py rename to sdk/python/kfp/deprecated/compiler/compiler.py index 9508b2b4e3b..0f983022ad2 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/deprecated/compiler/compiler.py @@ -23,20 +23,20 @@ import zipfile from typing import Callable, Set, List, Text, Dict, Tuple, Any, Union, Optional -import kfp -from kfp.dsl import _for_loop -from kfp.compiler import _data_passing_rewriter, v2_compat +import kfp.deprecated as kfp +from kfp.deprecated.dsl import _for_loop +from kfp.deprecated.compiler import _data_passing_rewriter, v2_compat -from kfp import dsl -from kfp.compiler._k8s_helper import convert_k8s_obj_to_json, sanitize_k8s_name -from kfp.compiler._op_to_template import _op_to_template, _process_obj -from kfp.compiler._default_transformers import add_pod_env, add_pod_labels +from kfp.deprecated import dsl +from kfp.deprecated.compiler._k8s_helper import convert_k8s_obj_to_json, sanitize_k8s_name +from kfp.deprecated.compiler._op_to_template import _op_to_template, _process_obj +from kfp.deprecated.compiler._default_transformers import add_pod_env, add_pod_labels -from kfp.components.structures import InputSpec -from kfp.components._yaml_utils import dump_yaml -from kfp.dsl._metadata import _extract_pipeline_metadata -from kfp.dsl._ops_group import OpsGroup -from kfp.dsl._pipeline_param import extract_pipelineparams_from_any, PipelineParam +from kfp.deprecated.components.structures import InputSpec +from kfp.deprecated.components._yaml_utils import dump_yaml +from kfp.deprecated.dsl._metadata import _extract_pipeline_metadata +from kfp.deprecated.dsl._ops_group import OpsGroup +from kfp.deprecated.dsl._pipeline_param import extract_pipelineparams_from_any, PipelineParam _SDK_VERSION_LABEL = 'pipelines.kubeflow.org/kfp_sdk_version' _SDK_ENV_LABEL = 'pipelines.kubeflow.org/pipeline-sdk-type' @@ -1163,7 +1163,7 @@ def compile(self, self._pipeline_name_param = dsl.PipelineParam( name='pipeline-name', value=f'pipeline/{pipeline_name}') - import kfp + import kfp.deprecated as kfp type_check_old_value = kfp.TYPE_CHECK compiling_for_v2_old_value = kfp.COMPILING_FOR_V2 kfp.COMPILING_FOR_V2 = self._mode in [ diff --git a/sdk/python/kfp/compiler/main.py b/sdk/python/kfp/deprecated/compiler/main.py similarity index 98% rename from sdk/python/kfp/compiler/main.py rename to sdk/python/kfp/deprecated/compiler/main.py index 3e11bee3c91..03bafa5dc95 100644 --- a/sdk/python/kfp/compiler/main.py +++ b/sdk/python/kfp/deprecated/compiler/main.py @@ -14,8 +14,8 @@ import argparse from typing import Optional -from kfp import dsl -import kfp.compiler +from kfp.deprecated import dsl +import kfp.deprecated.compiler import os import sys from deprecated.sphinx import deprecated diff --git a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml b/sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline.yaml similarity index 100% rename from sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline.yaml rename to sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline.yaml diff --git a/sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml b/sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml similarity index 100% rename from sdk/python/kfp/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml rename to sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml diff --git a/sdk/python/kfp/compiler/v2_compat.py b/sdk/python/kfp/deprecated/compiler/v2_compat.py similarity index 98% rename from sdk/python/kfp/compiler/v2_compat.py rename to sdk/python/kfp/deprecated/compiler/v2_compat.py index f6d05fdcfa1..00c513730f2 100644 --- a/sdk/python/kfp/compiler/v2_compat.py +++ b/sdk/python/kfp/deprecated/compiler/v2_compat.py @@ -16,8 +16,8 @@ import json from typing import Optional -from kfp import dsl -from kfp.compiler import _default_transformers +from kfp.deprecated import dsl +from kfp.deprecated.compiler import _default_transformers from kfp.pipeline_spec import pipeline_spec_pb2 from kfp.v2 import compiler from kubernetes import client as k8s_client diff --git a/sdk/python/kfp/compiler/v2_compatible_compiler_test.py b/sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py similarity index 98% rename from sdk/python/kfp/compiler/v2_compatible_compiler_test.py rename to sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py index 50f7ae0f2b3..e41be0566bf 100644 --- a/sdk/python/kfp/compiler/v2_compatible_compiler_test.py +++ b/sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py @@ -21,8 +21,8 @@ from typing import Callable, Dict import yaml -from kfp import compiler, components -from kfp import dsl as v1dsl +from kfp.deprecated import compiler, components +from kfp.deprecated import dsl as v1dsl from kfp.v2 import dsl from kfp.v2.dsl import Artifact, InputPath, OutputPath, component diff --git a/sdk/python/kfp/components/__init__.py b/sdk/python/kfp/deprecated/components/__init__.py similarity index 100% rename from sdk/python/kfp/components/__init__.py rename to sdk/python/kfp/deprecated/components/__init__.py diff --git a/sdk/python/kfp/components/_airflow_op.py b/sdk/python/kfp/deprecated/components/_airflow_op.py similarity index 100% rename from sdk/python/kfp/components/_airflow_op.py rename to sdk/python/kfp/deprecated/components/_airflow_op.py diff --git a/sdk/python/kfp/components/_component_store.py b/sdk/python/kfp/deprecated/components/_component_store.py similarity index 100% rename from sdk/python/kfp/components/_component_store.py rename to sdk/python/kfp/deprecated/components/_component_store.py diff --git a/sdk/python/kfp/components/_components.py b/sdk/python/kfp/deprecated/components/_components.py similarity index 100% rename from sdk/python/kfp/components/_components.py rename to sdk/python/kfp/deprecated/components/_components.py diff --git a/sdk/python/kfp/components/_data_passing.py b/sdk/python/kfp/deprecated/components/_data_passing.py similarity index 99% rename from sdk/python/kfp/components/_data_passing.py rename to sdk/python/kfp/deprecated/components/_data_passing.py index 9d8bf078368..285cfc05148 100644 --- a/sdk/python/kfp/components/_data_passing.py +++ b/sdk/python/kfp/deprecated/components/_data_passing.py @@ -23,7 +23,7 @@ from typing import Any, Callable, NamedTuple, Optional, Sequence, Type import warnings -from kfp.components import type_annotation_utils +from kfp.deprecated.components import type_annotation_utils Converter = NamedTuple('Converter', [ ('types', Sequence[Type]), diff --git a/sdk/python/kfp/components/_dynamic.py b/sdk/python/kfp/deprecated/components/_dynamic.py similarity index 100% rename from sdk/python/kfp/components/_dynamic.py rename to sdk/python/kfp/deprecated/components/_dynamic.py diff --git a/sdk/python/kfp/components/_key_value_store.py b/sdk/python/kfp/deprecated/components/_key_value_store.py similarity index 100% rename from sdk/python/kfp/components/_key_value_store.py rename to sdk/python/kfp/deprecated/components/_key_value_store.py diff --git a/sdk/python/kfp/components/_naming.py b/sdk/python/kfp/deprecated/components/_naming.py similarity index 100% rename from sdk/python/kfp/components/_naming.py rename to sdk/python/kfp/deprecated/components/_naming.py diff --git a/sdk/python/kfp/components/_python_op.py b/sdk/python/kfp/deprecated/components/_python_op.py similarity index 99% rename from sdk/python/kfp/components/_python_op.py rename to sdk/python/kfp/deprecated/components/_python_op.py index b3665a7a49f..d485e4b48da 100644 --- a/sdk/python/kfp/components/_python_op.py +++ b/sdk/python/kfp/deprecated/components/_python_op.py @@ -34,7 +34,7 @@ from ._naming import _make_name_unique_by_adding_index from .structures import * from . import _structures as structures -from kfp.components import type_annotation_utils +from kfp.deprecated.components import type_annotation_utils import inspect import itertools diff --git a/sdk/python/kfp/components/_python_to_graph_component.py b/sdk/python/kfp/deprecated/components/_python_to_graph_component.py similarity index 100% rename from sdk/python/kfp/components/_python_to_graph_component.py rename to sdk/python/kfp/deprecated/components/_python_to_graph_component.py diff --git a/sdk/python/kfp/components/_structures.py b/sdk/python/kfp/deprecated/components/_structures.py similarity index 100% rename from sdk/python/kfp/components/_structures.py rename to sdk/python/kfp/deprecated/components/_structures.py diff --git a/sdk/python/kfp/components/_yaml_utils.py b/sdk/python/kfp/deprecated/components/_yaml_utils.py similarity index 100% rename from sdk/python/kfp/components/_yaml_utils.py rename to sdk/python/kfp/deprecated/components/_yaml_utils.py diff --git a/sdk/python/kfp/components/modelbase.py b/sdk/python/kfp/deprecated/components/modelbase.py similarity index 100% rename from sdk/python/kfp/components/modelbase.py rename to sdk/python/kfp/deprecated/components/modelbase.py diff --git a/sdk/python/kfp/components/structures/__init__.py b/sdk/python/kfp/deprecated/components/structures/__init__.py similarity index 100% rename from sdk/python/kfp/components/structures/__init__.py rename to sdk/python/kfp/deprecated/components/structures/__init__.py diff --git a/sdk/python/kfp/components/structures/components.json_schema.json b/sdk/python/kfp/deprecated/components/structures/components.json_schema.json similarity index 100% rename from sdk/python/kfp/components/structures/components.json_schema.json rename to sdk/python/kfp/deprecated/components/structures/components.json_schema.json diff --git a/sdk/python/kfp/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/deprecated/components/structures/components.json_schema.outline.yaml similarity index 100% rename from sdk/python/kfp/components/structures/components.json_schema.outline.yaml rename to sdk/python/kfp/deprecated/components/structures/components.json_schema.outline.yaml diff --git a/sdk/python/kfp/components/structures/components.proto b/sdk/python/kfp/deprecated/components/structures/components.proto similarity index 100% rename from sdk/python/kfp/components/structures/components.proto rename to sdk/python/kfp/deprecated/components/structures/components.proto diff --git a/sdk/python/kfp/components/structures/generate_proto_code.sh b/sdk/python/kfp/deprecated/components/structures/generate_proto_code.sh similarity index 100% rename from sdk/python/kfp/components/structures/generate_proto_code.sh rename to sdk/python/kfp/deprecated/components/structures/generate_proto_code.sh diff --git a/sdk/python/kfp/components/type_annotation_utils.py b/sdk/python/kfp/deprecated/components/type_annotation_utils.py similarity index 100% rename from sdk/python/kfp/components/type_annotation_utils.py rename to sdk/python/kfp/deprecated/components/type_annotation_utils.py diff --git a/sdk/python/kfp/components/type_annotation_utils_test.py b/sdk/python/kfp/deprecated/components/type_annotation_utils_test.py similarity index 97% rename from sdk/python/kfp/components/type_annotation_utils_test.py rename to sdk/python/kfp/deprecated/components/type_annotation_utils_test.py index 4a9a815fd95..01fa6290404 100644 --- a/sdk/python/kfp/components/type_annotation_utils_test.py +++ b/sdk/python/kfp/deprecated/components/type_annotation_utils_test.py @@ -17,7 +17,7 @@ from absl.testing import parameterized from typing import Any, Dict, List, Optional -from kfp.components import type_annotation_utils +from kfp.deprecated.components import type_annotation_utils class TypeAnnotationUtilsTest(parameterized.TestCase): diff --git a/sdk/python/kfp/components_tests/__init__.py b/sdk/python/kfp/deprecated/components_tests/__init__.py similarity index 100% rename from sdk/python/kfp/components_tests/__init__.py rename to sdk/python/kfp/deprecated/components_tests/__init__.py diff --git a/sdk/python/kfp/components_tests/test_components.py b/sdk/python/kfp/deprecated/components_tests/test_components.py similarity index 99% rename from sdk/python/kfp/components_tests/test_components.py rename to sdk/python/kfp/deprecated/components_tests/test_components.py index fc33387d0d7..938ba2cec19 100644 --- a/sdk/python/kfp/components_tests/test_components.py +++ b/sdk/python/kfp/deprecated/components_tests/test_components.py @@ -18,10 +18,10 @@ from pathlib import Path from unittest import mock -from .. import components as comp -from ..components._components import _resolve_command_line_and_paths -from ..components._yaml_utils import load_yaml -from ..components.structures import ComponentSpec +from kfp.deprecated import components as comp +from kfp.deprecated.components._components import _resolve_command_line_and_paths +from kfp.deprecated.components._yaml_utils import load_yaml +from kfp.deprecated.components.structures import ComponentSpec class LoadComponentTestCase(unittest.TestCase): diff --git a/sdk/python/kfp/components_tests/test_data/component_with_0_inputs_and_2_outputs.component.yaml b/sdk/python/kfp/deprecated/components_tests/test_data/component_with_0_inputs_and_2_outputs.component.yaml similarity index 100% rename from sdk/python/kfp/components_tests/test_data/component_with_0_inputs_and_2_outputs.component.yaml rename to sdk/python/kfp/deprecated/components_tests/test_data/component_with_0_inputs_and_2_outputs.component.yaml diff --git a/sdk/python/kfp/components_tests/test_data/component_with_2_inputs_and_0_outputs.component.yaml b/sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_0_outputs.component.yaml similarity index 100% rename from sdk/python/kfp/components_tests/test_data/component_with_2_inputs_and_0_outputs.component.yaml rename to sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_0_outputs.component.yaml diff --git a/sdk/python/kfp/components_tests/test_data/component_with_2_inputs_and_2_outputs.component.yaml b/sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_2_outputs.component.yaml similarity index 100% rename from sdk/python/kfp/components_tests/test_data/component_with_2_inputs_and_2_outputs.component.yaml rename to sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_2_outputs.component.yaml diff --git a/sdk/python/kfp/components_tests/test_data/module1.py b/sdk/python/kfp/deprecated/components_tests/test_data/module1.py similarity index 100% rename from sdk/python/kfp/components_tests/test_data/module1.py rename to sdk/python/kfp/deprecated/components_tests/test_data/module1.py diff --git a/sdk/python/kfp/components_tests/test_data/module2_which_depends_on_module1.py b/sdk/python/kfp/deprecated/components_tests/test_data/module2_which_depends_on_module1.py similarity index 100% rename from sdk/python/kfp/components_tests/test_data/module2_which_depends_on_module1.py rename to sdk/python/kfp/deprecated/components_tests/test_data/module2_which_depends_on_module1.py diff --git a/sdk/python/kfp/components_tests/test_data/python_add.component.yaml b/sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.yaml similarity index 100% rename from sdk/python/kfp/components_tests/test_data/python_add.component.yaml rename to sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.yaml diff --git a/sdk/python/kfp/components_tests/test_data/python_add.component.zip b/sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.zip similarity index 100% rename from sdk/python/kfp/components_tests/test_data/python_add.component.zip rename to sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.zip diff --git a/sdk/python/kfp/components_tests/test_data/retail_product_stockout_prediction_pipeline.component.yaml b/sdk/python/kfp/deprecated/components_tests/test_data/retail_product_stockout_prediction_pipeline.component.yaml similarity index 100% rename from sdk/python/kfp/components_tests/test_data/retail_product_stockout_prediction_pipeline.component.yaml rename to sdk/python/kfp/deprecated/components_tests/test_data/retail_product_stockout_prediction_pipeline.component.yaml diff --git a/sdk/python/kfp/components_tests/test_data/retail_product_stockout_prediction_pipeline.py b/sdk/python/kfp/deprecated/components_tests/test_data/retail_product_stockout_prediction_pipeline.py similarity index 100% rename from sdk/python/kfp/components_tests/test_data/retail_product_stockout_prediction_pipeline.py rename to sdk/python/kfp/deprecated/components_tests/test_data/retail_product_stockout_prediction_pipeline.py diff --git a/sdk/python/kfp/components_tests/test_data_passing.py b/sdk/python/kfp/deprecated/components_tests/test_data_passing.py similarity index 95% rename from sdk/python/kfp/components_tests/test_data_passing.py rename to sdk/python/kfp/deprecated/components_tests/test_data_passing.py index 10b8c067231..94aff682c2d 100644 --- a/sdk/python/kfp/components_tests/test_data_passing.py +++ b/sdk/python/kfp/deprecated/components_tests/test_data_passing.py @@ -14,7 +14,7 @@ import unittest -from ..components import _data_passing +from kfp.deprecated.components import _data_passing class DataPassingTest(unittest.TestCase): diff --git a/sdk/python/kfp/components_tests/test_graph_components.py b/sdk/python/kfp/deprecated/components_tests/test_graph_components.py similarity index 96% rename from sdk/python/kfp/components_tests/test_graph_components.py rename to sdk/python/kfp/deprecated/components_tests/test_graph_components.py index f78d002bedd..056d5870dd1 100644 --- a/sdk/python/kfp/components_tests/test_graph_components.py +++ b/sdk/python/kfp/deprecated/components_tests/test_graph_components.py @@ -17,10 +17,10 @@ import unittest from pathlib import Path -from .. import components as comp -from ..components.structures import ComponentReference, ComponentSpec, ContainerSpec, GraphInputReference, GraphSpec, InputSpec, InputValuePlaceholder, GraphImplementation, OutputPathPlaceholder, OutputSpec, TaskOutputArgument, TaskSpec +from kfp.deprecated import components as comp +from kfp.deprecated.components.structures import ComponentReference, ComponentSpec, ContainerSpec, GraphInputReference, GraphSpec, InputSpec, InputValuePlaceholder, GraphImplementation, OutputPathPlaceholder, OutputSpec, TaskOutputArgument, TaskSpec -from ..components._yaml_utils import load_yaml +from kfp.deprecated.components._yaml_utils import load_yaml class GraphComponentTestCase(unittest.TestCase): diff --git a/sdk/python/kfp/components_tests/test_python_op.py b/sdk/python/kfp/deprecated/components_tests/test_python_op.py similarity index 99% rename from sdk/python/kfp/components_tests/test_python_op.py rename to sdk/python/kfp/deprecated/components_tests/test_python_op.py index f139ddc5920..ca56edd3c1d 100644 --- a/sdk/python/kfp/components_tests/test_python_op.py +++ b/sdk/python/kfp/deprecated/components_tests/test_python_op.py @@ -20,11 +20,11 @@ from pathlib import Path from typing import Callable, NamedTuple, Sequence -from .. import components as comp -from ..components import (InputBinaryFile, InputPath, InputTextFile, +from kfp.deprecated import components as comp +from kfp.deprecated.components import (InputBinaryFile, InputPath, InputTextFile, OutputBinaryFile, OutputPath, OutputTextFile) -from ..components._components import _resolve_command_line_and_paths -from ..components.structures import InputSpec, OutputSpec +from kfp.deprecated.components._components import _resolve_command_line_and_paths +from kfp.deprecated.components.structures import InputSpec, OutputSpec def add_two_numbers(a: float, b: float) -> float: @@ -302,7 +302,7 @@ def test_func_to_container_op_call_other_func_global(self): self.helper_test_2_in_1_out_component_using_local_call(func, op) def test_func_to_container_op_with_imported_func(self): - from .test_data.module1 import \ + from kfp.deprecated.components_tests.test_data.module1 import \ module_func_with_deps as module1_func_with_deps func = module1_func_with_deps op = comp.func_to_container_op(func, use_code_pickling=True) @@ -310,7 +310,7 @@ def test_func_to_container_op_with_imported_func(self): self.helper_test_2_in_1_out_component_using_local_call(func, op) def test_func_to_container_op_with_imported_func2(self): - from .test_data import module1, module2_which_depends_on_module1 + from kfp.deprecated.components_tests.test_data import module1, module2_which_depends_on_module1 func = module2_which_depends_on_module1.module2_func_with_deps op = comp.func_to_container_op( func, diff --git a/sdk/python/kfp/components_tests/test_python_pipeline_to_graph_component.py b/sdk/python/kfp/deprecated/components_tests/test_python_pipeline_to_graph_component.py similarity index 91% rename from sdk/python/kfp/components_tests/test_python_pipeline_to_graph_component.py rename to sdk/python/kfp/deprecated/components_tests/test_python_pipeline_to_graph_component.py index 2527909d8f1..f6367f49fcc 100644 --- a/sdk/python/kfp/components_tests/test_python_pipeline_to_graph_component.py +++ b/sdk/python/kfp/deprecated/components_tests/test_python_pipeline_to_graph_component.py @@ -18,8 +18,8 @@ from collections import OrderedDict from pathlib import Path -from .. import components as comp -from ..components._python_to_graph_component import create_graph_component_spec_from_pipeline_func +from kfp.deprecated import components as comp +from kfp.deprecated.components._python_to_graph_component import create_graph_component_spec_from_pipeline_func class PythonPipelineToGraphComponentTestCase(unittest.TestCase): @@ -65,7 +65,7 @@ def pipeline1(pipeline_param_1: int): def test_create_component_from_real_pipeline_retail_product_stockout_prediction( self): - from .test_data.retail_product_stockout_prediction_pipeline import retail_product_stockout_prediction_pipeline + from kfp.deprecated.components_tests.test_data.retail_product_stockout_prediction_pipeline import retail_product_stockout_prediction_pipeline graph_component = create_graph_component_spec_from_pipeline_func( retail_product_stockout_prediction_pipeline) diff --git a/sdk/python/kfp/components_tests/test_structure_model_base.py b/sdk/python/kfp/deprecated/components_tests/test_structure_model_base.py similarity index 99% rename from sdk/python/kfp/components_tests/test_structure_model_base.py rename to sdk/python/kfp/deprecated/components_tests/test_structure_model_base.py index d3af3af95f9..d317ecb5780 100644 --- a/sdk/python/kfp/components_tests/test_structure_model_base.py +++ b/sdk/python/kfp/deprecated/components_tests/test_structure_model_base.py @@ -18,7 +18,7 @@ from pathlib import Path from typing import List, Dict, Union, Optional -from ..components.modelbase import ModelBase +from kfp.deprecated.components.modelbase import ModelBase class TestModel1(ModelBase): diff --git a/sdk/python/kfp/containers/__init__.py b/sdk/python/kfp/deprecated/containers/__init__.py similarity index 100% rename from sdk/python/kfp/containers/__init__.py rename to sdk/python/kfp/deprecated/containers/__init__.py diff --git a/sdk/python/kfp/containers/_build_image_api.py b/sdk/python/kfp/deprecated/containers/_build_image_api.py similarity index 100% rename from sdk/python/kfp/containers/_build_image_api.py rename to sdk/python/kfp/deprecated/containers/_build_image_api.py diff --git a/sdk/python/kfp/containers/_cache.py b/sdk/python/kfp/deprecated/containers/_cache.py similarity index 100% rename from sdk/python/kfp/containers/_cache.py rename to sdk/python/kfp/deprecated/containers/_cache.py diff --git a/sdk/python/kfp/containers/_component_builder.py b/sdk/python/kfp/deprecated/containers/_component_builder.py similarity index 98% rename from sdk/python/kfp/containers/_component_builder.py rename to sdk/python/kfp/deprecated/containers/_component_builder.py index e9dff7d1fc0..4e65dccd68b 100644 --- a/sdk/python/kfp/containers/_component_builder.py +++ b/sdk/python/kfp/deprecated/containers/_component_builder.py @@ -25,11 +25,11 @@ from ..components._components import _create_task_factory_from_component_spec from ..components._python_op import _func_to_component_spec from ._container_builder import ContainerBuilder -from kfp import components -from kfp import dsl -from kfp.components import _components -from kfp.components import _structures -from kfp.containers import entrypoint +from kfp.deprecated import components +from kfp.deprecated import dsl +from kfp.deprecated.components import _components +from kfp.deprecated.components import _structures +from kfp.deprecated.containers import entrypoint V2_COMPONENT_ANNOTATION = 'pipelines.kubeflow.org/component_v2' _PROGRAM_LAUNCHER_CMD = 'program_path=$(mktemp)\nprintf "%s" "$0" > ' \ diff --git a/sdk/python/kfp/containers/_container_builder.py b/sdk/python/kfp/deprecated/containers/_container_builder.py similarity index 100% rename from sdk/python/kfp/containers/_container_builder.py rename to sdk/python/kfp/deprecated/containers/_container_builder.py diff --git a/sdk/python/kfp/containers/_gcs_helper.py b/sdk/python/kfp/deprecated/containers/_gcs_helper.py similarity index 100% rename from sdk/python/kfp/containers/_gcs_helper.py rename to sdk/python/kfp/deprecated/containers/_gcs_helper.py diff --git a/sdk/python/kfp/containers/_k8s_job_helper.py b/sdk/python/kfp/deprecated/containers/_k8s_job_helper.py similarity index 100% rename from sdk/python/kfp/containers/_k8s_job_helper.py rename to sdk/python/kfp/deprecated/containers/_k8s_job_helper.py diff --git a/sdk/python/kfp/containers/entrypoint.py b/sdk/python/kfp/deprecated/containers/entrypoint.py similarity index 98% rename from sdk/python/kfp/containers/entrypoint.py rename to sdk/python/kfp/deprecated/containers/entrypoint.py index c8418bb3b6b..823a9524ac5 100644 --- a/sdk/python/kfp/containers/entrypoint.py +++ b/sdk/python/kfp/deprecated/containers/entrypoint.py @@ -19,9 +19,9 @@ from google.protobuf import json_format import os -from kfp.containers import _gcs_helper -from kfp.containers import entrypoint_utils -from kfp.dsl import artifact +from kfp.deprecated.containers import _gcs_helper +from kfp.deprecated.containers import entrypoint_utils +from kfp.deprecated.dsl import artifact from kfp.pipeline_spec import pipeline_spec_pb2 FN_SOURCE = 'ml/main.py' diff --git a/sdk/python/kfp/containers/entrypoint_utils.py b/sdk/python/kfp/deprecated/containers/entrypoint_utils.py similarity index 98% rename from sdk/python/kfp/containers/entrypoint_utils.py rename to sdk/python/kfp/deprecated/containers/entrypoint_utils.py index 736cb43142b..4a7f30298aa 100644 --- a/sdk/python/kfp/containers/entrypoint_utils.py +++ b/sdk/python/kfp/deprecated/containers/entrypoint_utils.py @@ -18,10 +18,10 @@ from typing import Callable, Dict, Optional, Union from google.protobuf import json_format -from kfp.components import _python_op -from kfp.containers import _gcs_helper +from kfp.deprecated.components import _python_op +from kfp.deprecated.containers import _gcs_helper from kfp.pipeline_spec import pipeline_spec_pb2 -from kfp.dsl import artifact +from kfp.deprecated.dsl import artifact # If path starts with one of those, consider files are in remote filesystem. _REMOTE_FS_PREFIX = ['gs://', 'hdfs://', 's3://'] diff --git a/sdk/python/kfp/containers_tests/__init__.py b/sdk/python/kfp/deprecated/containers_tests/__init__.py similarity index 100% rename from sdk/python/kfp/containers_tests/__init__.py rename to sdk/python/kfp/deprecated/containers_tests/__init__.py diff --git a/sdk/python/kfp/containers_tests/component_builder_test.py b/sdk/python/kfp/deprecated/containers_tests/component_builder_test.py similarity index 93% rename from sdk/python/kfp/containers_tests/component_builder_test.py rename to sdk/python/kfp/deprecated/containers_tests/component_builder_test.py index 6623cec76a7..eb3c1d2d458 100644 --- a/sdk/python/kfp/containers_tests/component_builder_test.py +++ b/sdk/python/kfp/deprecated/containers_tests/component_builder_test.py @@ -18,9 +18,9 @@ import unittest from unittest import mock -from kfp.containers import _component_builder -from kfp.containers import _container_builder -from kfp import components +from kfp.deprecated.containers import _component_builder +from kfp.deprecated.containers import _container_builder +from kfp.deprecated import components _TEST_TARGET_IMAGE = 'gcr.io/my-project/my-image' _TEST_STAGING_LOCATION = 'gs://my-project/tmp' diff --git a/sdk/python/kfp/containers_tests/test_build_image_api.py b/sdk/python/kfp/deprecated/containers_tests/test_build_image_api.py similarity index 98% rename from sdk/python/kfp/containers_tests/test_build_image_api.py rename to sdk/python/kfp/deprecated/containers_tests/test_build_image_api.py index d65f94b884b..cffb4d2b783 100644 --- a/sdk/python/kfp/containers_tests/test_build_image_api.py +++ b/sdk/python/kfp/deprecated/containers_tests/test_build_image_api.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Callable -from kfp.containers import build_image_from_working_dir +from kfp.deprecated.containers import build_image_from_working_dir class MockImageBuilder: @@ -102,7 +102,7 @@ def file_paths_check(file_paths): def test_image_cache(self): builder = InvocationCountingDummyImageBuilder() - from kfp.containers._cache import clear_cache + from kfp.deprecated.containers._cache import clear_cache clear_cache('build_image_from_working_dir') self.assertEqual(builder.invocations_count, 0) diff --git a/sdk/python/kfp/containers_tests/testdata/__init__.py b/sdk/python/kfp/deprecated/containers_tests/testdata/__init__.py similarity index 100% rename from sdk/python/kfp/containers_tests/testdata/__init__.py rename to sdk/python/kfp/deprecated/containers_tests/testdata/__init__.py diff --git a/sdk/python/kfp/containers_tests/testdata/executor_output.json b/sdk/python/kfp/deprecated/containers_tests/testdata/executor_output.json similarity index 100% rename from sdk/python/kfp/containers_tests/testdata/executor_output.json rename to sdk/python/kfp/deprecated/containers_tests/testdata/executor_output.json diff --git a/sdk/python/kfp/containers_tests/testdata/expected_component.yaml b/sdk/python/kfp/deprecated/containers_tests/testdata/expected_component.yaml similarity index 100% rename from sdk/python/kfp/containers_tests/testdata/expected_component.yaml rename to sdk/python/kfp/deprecated/containers_tests/testdata/expected_component.yaml diff --git a/sdk/python/kfp/containers_tests/testdata/main.py b/sdk/python/kfp/deprecated/containers_tests/testdata/main.py similarity index 94% rename from sdk/python/kfp/containers_tests/testdata/main.py rename to sdk/python/kfp/deprecated/containers_tests/testdata/main.py index 265d076c49d..e72028a8a7a 100644 --- a/sdk/python/kfp/containers_tests/testdata/main.py +++ b/sdk/python/kfp/deprecated/containers_tests/testdata/main.py @@ -13,9 +13,9 @@ # limitations under the License. """User module under test.""" from typing import NamedTuple -from kfp import components -from kfp.dsl import artifact -from kfp.dsl import ontology_artifacts +from kfp.deprecated import components +from kfp.deprecated.dsl import artifact +from kfp.deprecated.dsl import ontology_artifacts def test_func( diff --git a/sdk/python/kfp/containers_tests/testdata/pipeline_source.py b/sdk/python/kfp/deprecated/containers_tests/testdata/pipeline_source.py similarity index 100% rename from sdk/python/kfp/containers_tests/testdata/pipeline_source.py rename to sdk/python/kfp/deprecated/containers_tests/testdata/pipeline_source.py diff --git a/sdk/python/kfp/containers_tests/testdata/test_source.py b/sdk/python/kfp/deprecated/containers_tests/testdata/test_source.py similarity index 100% rename from sdk/python/kfp/containers_tests/testdata/test_source.py rename to sdk/python/kfp/deprecated/containers_tests/testdata/test_source.py diff --git a/sdk/python/kfp/dsl/__init__.py b/sdk/python/kfp/deprecated/dsl/__init__.py similarity index 100% rename from sdk/python/kfp/dsl/__init__.py rename to sdk/python/kfp/deprecated/dsl/__init__.py diff --git a/sdk/python/kfp/dsl/_component.py b/sdk/python/kfp/deprecated/dsl/_component.py similarity index 99% rename from sdk/python/kfp/dsl/_component.py rename to sdk/python/kfp/deprecated/dsl/_component.py index e4cdd3c9d49..5a08038072b 100644 --- a/sdk/python/kfp/dsl/_component.py +++ b/sdk/python/kfp/deprecated/dsl/_component.py @@ -17,7 +17,7 @@ from ._pipeline_param import PipelineParam from .types import check_types, InconsistentTypeException from ._ops_group import Graph -import kfp +import kfp.deprecated @deprecated( diff --git a/sdk/python/kfp/dsl/_component_bridge.py b/sdk/python/kfp/deprecated/dsl/_component_bridge.py similarity index 98% rename from sdk/python/kfp/dsl/_component_bridge.py rename to sdk/python/kfp/deprecated/dsl/_component_bridge.py index c61a6cb0d1f..6c85029e4f7 100644 --- a/sdk/python/kfp/dsl/_component_bridge.py +++ b/sdk/python/kfp/deprecated/dsl/_component_bridge.py @@ -19,17 +19,17 @@ import pathlib from typing import Any, Mapping, Optional -import kfp -from kfp.components import _structures, _data_passing -from kfp.components import _components -from kfp.components import _naming -from kfp import dsl -from kfp.dsl import _container_op -from kfp.dsl import _for_loop -from kfp.dsl import _pipeline_param -from kfp.dsl import component_spec as dsl_component_spec -from kfp.dsl import dsl_utils -from kfp.dsl import types +import kfp.deprecated +from kfp.deprecated.components import _structures, _data_passing +from kfp.deprecated.components import _components +from kfp.deprecated.components import _naming +from kfp.deprecated import dsl +from kfp.deprecated.dsl import _container_op +from kfp.deprecated.dsl import _for_loop +from kfp.deprecated.dsl import _pipeline_param +from kfp.deprecated.dsl import component_spec as dsl_component_spec +from kfp.deprecated.dsl import dsl_utils +from kfp.deprecated.dsl import types from kfp.pipeline_spec import pipeline_spec_pb2 from kfp.v2.components.types import type_utils diff --git a/sdk/python/kfp/dsl/_container_op.py b/sdk/python/kfp/deprecated/dsl/_container_op.py similarity index 99% rename from sdk/python/kfp/dsl/_container_op.py rename to sdk/python/kfp/deprecated/dsl/_container_op.py index 52913881e21..6acb1605b64 100644 --- a/sdk/python/kfp/dsl/_container_op.py +++ b/sdk/python/kfp/deprecated/dsl/_container_op.py @@ -17,9 +17,9 @@ from typing import (Any, Callable, Dict, List, Optional, Sequence, Tuple, TypeVar, Union) -import kfp -from kfp.components import _components, _structures -from kfp.dsl import _pipeline_param, dsl_utils +import kfp.deprecated +from kfp.deprecated.components import _components, _structures +from kfp.deprecated.dsl import _pipeline_param, dsl_utils from kfp.pipeline_spec import pipeline_spec_pb2 from kubernetes.client import V1Affinity, V1Toleration from kubernetes.client.models import (V1Container, V1ContainerPort, diff --git a/sdk/python/kfp/dsl/_container_op_test.py b/sdk/python/kfp/deprecated/dsl/_container_op_test.py similarity index 94% rename from sdk/python/kfp/dsl/_container_op_test.py rename to sdk/python/kfp/deprecated/dsl/_container_op_test.py index 51a79172d89..43d0bd2fd1c 100644 --- a/sdk/python/kfp/dsl/_container_op_test.py +++ b/sdk/python/kfp/deprecated/dsl/_container_op_test.py @@ -14,8 +14,8 @@ """Tests for kfp.v2.dsl.container_op.""" import unittest -from kfp.dsl import _container_op -from kfp.pipeline_spec import pipeline_spec_pb2 +from kfp.deprecated.dsl import _container_op +from kfp.deprecated.pipeline_spec import pipeline_spec_pb2 from google.protobuf import text_format from google.protobuf import json_format diff --git a/sdk/python/kfp/dsl/_for_loop.py b/sdk/python/kfp/deprecated/dsl/_for_loop.py similarity index 99% rename from sdk/python/kfp/dsl/_for_loop.py rename to sdk/python/kfp/deprecated/dsl/_for_loop.py index f7db635f8f8..2650409e55f 100644 --- a/sdk/python/kfp/dsl/_for_loop.py +++ b/sdk/python/kfp/deprecated/dsl/_for_loop.py @@ -14,8 +14,8 @@ import re from typing import Any, Dict, List, Optional, Tuple, Union -from kfp import dsl -from kfp.dsl import _pipeline_param +from kfp.deprecated import dsl +from kfp.deprecated.dsl import _pipeline_param ItemList = List[Union[int, float, str, Dict[str, Any]]] diff --git a/sdk/python/kfp/dsl/_metadata.py b/sdk/python/kfp/deprecated/dsl/_metadata.py similarity index 97% rename from sdk/python/kfp/dsl/_metadata.py rename to sdk/python/kfp/deprecated/dsl/_metadata.py index 291c8c76439..acefc08cef4 100644 --- a/sdk/python/kfp/dsl/_metadata.py +++ b/sdk/python/kfp/deprecated/dsl/_metadata.py @@ -93,6 +93,6 @@ def _extract_pipeline_metadata(func): # Only validating non-serialized values validate(instance=arg_default, schema=schema_object) - from kfp.components._python_op import _extract_component_interface + from kfp.deprecated.components._python_op import _extract_component_interface component_spec = _extract_component_interface(func) return component_spec diff --git a/sdk/python/kfp/dsl/_ops_group.py b/sdk/python/kfp/deprecated/dsl/_ops_group.py similarity index 99% rename from sdk/python/kfp/dsl/_ops_group.py rename to sdk/python/kfp/deprecated/dsl/_ops_group.py index a74ebb6c8ca..8d19fcd999f 100644 --- a/sdk/python/kfp/dsl/_ops_group.py +++ b/sdk/python/kfp/deprecated/dsl/_ops_group.py @@ -14,7 +14,7 @@ from typing import Union import uuid -from kfp.dsl import _for_loop, _pipeline_param +from kfp.deprecated.dsl import _for_loop, _pipeline_param from . import _container_op from . import _pipeline diff --git a/sdk/python/kfp/dsl/_pipeline.py b/sdk/python/kfp/deprecated/dsl/_pipeline.py similarity index 97% rename from sdk/python/kfp/dsl/_pipeline.py rename to sdk/python/kfp/deprecated/dsl/_pipeline.py index e34b348e759..68fe5619472 100644 --- a/sdk/python/kfp/dsl/_pipeline.py +++ b/sdk/python/kfp/deprecated/dsl/_pipeline.py @@ -16,12 +16,12 @@ from typing import Callable, Optional, Union from kubernetes.client.models import V1PodDNSConfig -from kfp.dsl import _container_op -from kfp.dsl import _resource_op -from kfp.dsl import _ops_group -from kfp.dsl import _component_bridge -from kfp.components import _components -from kfp.components import _naming +from kfp.deprecated.dsl import _container_op +from kfp.deprecated.dsl import _resource_op +from kfp.deprecated.dsl import _ops_group +from kfp.deprecated.dsl import _component_bridge +from kfp.deprecated.components import _components +from kfp.deprecated.components import _naming import sys # This handler is called whenever the @pipeline decorator is applied. diff --git a/sdk/python/kfp/dsl/_pipeline_param.py b/sdk/python/kfp/deprecated/dsl/_pipeline_param.py similarity index 100% rename from sdk/python/kfp/dsl/_pipeline_param.py rename to sdk/python/kfp/deprecated/dsl/_pipeline_param.py diff --git a/sdk/python/kfp/dsl/_pipeline_volume.py b/sdk/python/kfp/deprecated/dsl/_pipeline_volume.py similarity index 100% rename from sdk/python/kfp/dsl/_pipeline_volume.py rename to sdk/python/kfp/deprecated/dsl/_pipeline_volume.py diff --git a/sdk/python/kfp/dsl/_resource_op.py b/sdk/python/kfp/deprecated/dsl/_resource_op.py similarity index 100% rename from sdk/python/kfp/dsl/_resource_op.py rename to sdk/python/kfp/deprecated/dsl/_resource_op.py diff --git a/sdk/python/kfp/dsl/_volume_op.py b/sdk/python/kfp/deprecated/dsl/_volume_op.py similarity index 100% rename from sdk/python/kfp/dsl/_volume_op.py rename to sdk/python/kfp/deprecated/dsl/_volume_op.py diff --git a/sdk/python/kfp/dsl/_volume_snapshot_op.py b/sdk/python/kfp/deprecated/dsl/_volume_snapshot_op.py similarity index 100% rename from sdk/python/kfp/dsl/_volume_snapshot_op.py rename to sdk/python/kfp/deprecated/dsl/_volume_snapshot_op.py diff --git a/sdk/python/kfp/dsl/artifact.py b/sdk/python/kfp/deprecated/dsl/artifact.py similarity index 98% rename from sdk/python/kfp/dsl/artifact.py rename to sdk/python/kfp/deprecated/dsl/artifact.py index c74b7da1237..9e04bb4987e 100644 --- a/sdk/python/kfp/dsl/artifact.py +++ b/sdk/python/kfp/deprecated/dsl/artifact.py @@ -24,8 +24,8 @@ from google.protobuf import json_format from google.protobuf import struct_pb2 from kfp.pipeline_spec import pipeline_spec_pb2 -from kfp.dsl import serialization_utils -from kfp.dsl import artifact_utils +from kfp.deprecated.dsl import serialization_utils +from kfp.deprecated.dsl import artifact_utils KFP_ARTIFACT_ONTOLOGY_MODULE = 'kfp.dsl.ontology_artifacts' DEFAULT_ARTIFACT_SCHEMA = 'title: kfp.Artifact\ntype: object\nproperties:\n' diff --git a/sdk/python/kfp/dsl/artifact_utils.py b/sdk/python/kfp/deprecated/dsl/artifact_utils.py similarity index 100% rename from sdk/python/kfp/dsl/artifact_utils.py rename to sdk/python/kfp/deprecated/dsl/artifact_utils.py diff --git a/sdk/python/kfp/dsl/component_spec.py b/sdk/python/kfp/deprecated/dsl/component_spec.py similarity index 99% rename from sdk/python/kfp/dsl/component_spec.py rename to sdk/python/kfp/deprecated/dsl/component_spec.py index 57d70e0f93c..4b8de8be779 100644 --- a/sdk/python/kfp/dsl/component_spec.py +++ b/sdk/python/kfp/deprecated/dsl/component_spec.py @@ -15,8 +15,8 @@ from typing import List, Optional, Tuple, Union -from kfp.components import _structures as structures -from kfp.dsl import _for_loop, _pipeline_param, dsl_utils +from kfp.deprecated.components import _structures as structures +from kfp.deprecated.dsl import _for_loop, _pipeline_param, dsl_utils from kfp.pipeline_spec import pipeline_spec_pb2 from kfp.v2.components.types import type_utils diff --git a/sdk/python/kfp/dsl/component_spec_test.py b/sdk/python/kfp/deprecated/dsl/component_spec_test.py similarity index 99% rename from sdk/python/kfp/dsl/component_spec_test.py rename to sdk/python/kfp/deprecated/dsl/component_spec_test.py index 73a49923eeb..856615c3202 100644 --- a/sdk/python/kfp/dsl/component_spec_test.py +++ b/sdk/python/kfp/deprecated/dsl/component_spec_test.py @@ -15,9 +15,9 @@ from absl.testing import parameterized -from kfp.components import _structures as structures -from kfp.dsl import _pipeline_param -from kfp.dsl import component_spec as dsl_component_spec +from kfp.deprecated.components import _structures as structures +from kfp.deprecated.dsl import _pipeline_param +from kfp.deprecated.dsl import component_spec as dsl_component_spec from kfp.pipeline_spec import pipeline_spec_pb2 from google.protobuf import json_format diff --git a/sdk/python/kfp/dsl/data_passing_methods.py b/sdk/python/kfp/deprecated/dsl/data_passing_methods.py similarity index 100% rename from sdk/python/kfp/dsl/data_passing_methods.py rename to sdk/python/kfp/deprecated/dsl/data_passing_methods.py diff --git a/sdk/python/kfp/dsl/dsl_utils.py b/sdk/python/kfp/deprecated/dsl/dsl_utils.py similarity index 99% rename from sdk/python/kfp/dsl/dsl_utils.py rename to sdk/python/kfp/deprecated/dsl/dsl_utils.py index 0a5bddb85c5..f78ae748018 100644 --- a/sdk/python/kfp/dsl/dsl_utils.py +++ b/sdk/python/kfp/deprecated/dsl/dsl_utils.py @@ -16,7 +16,7 @@ import re from typing import Callable, List, Optional, Union -from kfp.components import _structures +from kfp.deprecated.components import _structures from kfp.pipeline_spec import pipeline_spec_pb2 _COMPONENT_NAME_PREFIX = 'comp-' diff --git a/sdk/python/kfp/dsl/dsl_utils_test.py b/sdk/python/kfp/deprecated/dsl/dsl_utils_test.py similarity index 98% rename from sdk/python/kfp/dsl/dsl_utils_test.py rename to sdk/python/kfp/deprecated/dsl/dsl_utils_test.py index 68460a9609b..c78198f4ed8 100644 --- a/sdk/python/kfp/dsl/dsl_utils_test.py +++ b/sdk/python/kfp/deprecated/dsl/dsl_utils_test.py @@ -15,7 +15,7 @@ import unittest -from kfp.dsl import dsl_utils +from kfp.deprecated.dsl import dsl_utils from kfp.pipeline_spec import pipeline_spec_pb2 from google.protobuf import json_format diff --git a/sdk/python/kfp/dsl/extensions/__init__.py b/sdk/python/kfp/deprecated/dsl/extensions/__init__.py similarity index 100% rename from sdk/python/kfp/dsl/extensions/__init__.py rename to sdk/python/kfp/deprecated/dsl/extensions/__init__.py diff --git a/sdk/python/kfp/dsl/extensions/kubernetes.py b/sdk/python/kfp/deprecated/dsl/extensions/kubernetes.py similarity index 100% rename from sdk/python/kfp/dsl/extensions/kubernetes.py rename to sdk/python/kfp/deprecated/dsl/extensions/kubernetes.py diff --git a/sdk/python/kfp/dsl/io_types.py b/sdk/python/kfp/deprecated/dsl/io_types.py similarity index 100% rename from sdk/python/kfp/dsl/io_types.py rename to sdk/python/kfp/deprecated/dsl/io_types.py diff --git a/sdk/python/kfp/dsl/metrics_utils.py b/sdk/python/kfp/deprecated/dsl/metrics_utils.py similarity index 99% rename from sdk/python/kfp/dsl/metrics_utils.py rename to sdk/python/kfp/deprecated/dsl/metrics_utils.py index 581545df490..b7a2a9ece50 100644 --- a/sdk/python/kfp/dsl/metrics_utils.py +++ b/sdk/python/kfp/deprecated/dsl/metrics_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.dsl import artifact_utils +from kfp.deprecated.dsl import artifact_utils from typing import Any, List diff --git a/sdk/python/kfp/dsl/metrics_utils_test.py b/sdk/python/kfp/deprecated/dsl/metrics_utils_test.py similarity index 98% rename from sdk/python/kfp/dsl/metrics_utils_test.py rename to sdk/python/kfp/deprecated/dsl/metrics_utils_test.py index 7c3c4b6ed21..6790aa5095e 100644 --- a/sdk/python/kfp/dsl/metrics_utils_test.py +++ b/sdk/python/kfp/deprecated/dsl/metrics_utils_test.py @@ -22,7 +22,7 @@ from google.protobuf import json_format from google.protobuf.struct_pb2 import Struct -from kfp.dsl import metrics_utils +from kfp.deprecated.dsl import metrics_utils from google.protobuf import json_format diff --git a/sdk/python/kfp/dsl/serialization_utils.py b/sdk/python/kfp/deprecated/dsl/serialization_utils.py similarity index 100% rename from sdk/python/kfp/dsl/serialization_utils.py rename to sdk/python/kfp/deprecated/dsl/serialization_utils.py diff --git a/sdk/python/kfp/dsl/serialization_utils_test.py b/sdk/python/kfp/deprecated/dsl/serialization_utils_test.py similarity index 95% rename from sdk/python/kfp/dsl/serialization_utils_test.py rename to sdk/python/kfp/deprecated/dsl/serialization_utils_test.py index 24c802d1b1e..cbee21e608d 100644 --- a/sdk/python/kfp/dsl/serialization_utils_test.py +++ b/sdk/python/kfp/deprecated/dsl/serialization_utils_test.py @@ -14,7 +14,7 @@ """Tests for kfp.dsl.serialization_utils module.""" import unittest -from kfp.dsl import serialization_utils +from kfp.deprecated.dsl import serialization_utils _DICT_DATA = { 'int1': 1, diff --git a/sdk/python/kfp/dsl/test_data/expected_bulk_loaded_confusion_matrix.json b/sdk/python/kfp/deprecated/dsl/test_data/expected_bulk_loaded_confusion_matrix.json similarity index 100% rename from sdk/python/kfp/dsl/test_data/expected_bulk_loaded_confusion_matrix.json rename to sdk/python/kfp/deprecated/dsl/test_data/expected_bulk_loaded_confusion_matrix.json diff --git a/sdk/python/kfp/dsl/test_data/expected_confusion_matrix.json b/sdk/python/kfp/deprecated/dsl/test_data/expected_confusion_matrix.json similarity index 100% rename from sdk/python/kfp/dsl/test_data/expected_confusion_matrix.json rename to sdk/python/kfp/deprecated/dsl/test_data/expected_confusion_matrix.json diff --git a/sdk/python/kfp/dsl/type_schemas/classification_metrics.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/classification_metrics.yaml similarity index 100% rename from sdk/python/kfp/dsl/type_schemas/classification_metrics.yaml rename to sdk/python/kfp/deprecated/dsl/type_schemas/classification_metrics.yaml diff --git a/sdk/python/kfp/dsl/type_schemas/confidence_metrics.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/confidence_metrics.yaml similarity index 100% rename from sdk/python/kfp/dsl/type_schemas/confidence_metrics.yaml rename to sdk/python/kfp/deprecated/dsl/type_schemas/confidence_metrics.yaml diff --git a/sdk/python/kfp/dsl/type_schemas/confusion_matrix.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/confusion_matrix.yaml similarity index 100% rename from sdk/python/kfp/dsl/type_schemas/confusion_matrix.yaml rename to sdk/python/kfp/deprecated/dsl/type_schemas/confusion_matrix.yaml diff --git a/sdk/python/kfp/dsl/type_schemas/dataset.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/dataset.yaml similarity index 100% rename from sdk/python/kfp/dsl/type_schemas/dataset.yaml rename to sdk/python/kfp/deprecated/dsl/type_schemas/dataset.yaml diff --git a/sdk/python/kfp/dsl/type_schemas/metrics.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/metrics.yaml similarity index 100% rename from sdk/python/kfp/dsl/type_schemas/metrics.yaml rename to sdk/python/kfp/deprecated/dsl/type_schemas/metrics.yaml diff --git a/sdk/python/kfp/dsl/type_schemas/model.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/model.yaml similarity index 100% rename from sdk/python/kfp/dsl/type_schemas/model.yaml rename to sdk/python/kfp/deprecated/dsl/type_schemas/model.yaml diff --git a/sdk/python/kfp/dsl/type_schemas/sliced_classification_metrics.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/sliced_classification_metrics.yaml similarity index 100% rename from sdk/python/kfp/dsl/type_schemas/sliced_classification_metrics.yaml rename to sdk/python/kfp/deprecated/dsl/type_schemas/sliced_classification_metrics.yaml diff --git a/sdk/python/kfp/dsl/type_utils.py b/sdk/python/kfp/deprecated/dsl/type_utils.py similarity index 100% rename from sdk/python/kfp/dsl/type_utils.py rename to sdk/python/kfp/deprecated/dsl/type_utils.py diff --git a/sdk/python/kfp/dsl/types.py b/sdk/python/kfp/deprecated/dsl/types.py similarity index 99% rename from sdk/python/kfp/dsl/types.py rename to sdk/python/kfp/deprecated/dsl/types.py index 152da36c6eb..e4940142cfa 100644 --- a/sdk/python/kfp/dsl/types.py +++ b/sdk/python/kfp/deprecated/dsl/types.py @@ -157,7 +157,7 @@ def verify_type_compatibility(given_type: TypeSpecType, error_text = error_message_prefix + ( 'Argument type "{}" is incompatible with the input type "{}"' ).format(str(given_type), str(expected_type)) - import kfp + import kfp.deprecated if kfp.TYPE_CHECK: raise InconsistentTypeException(error_text) else: diff --git a/sdk/python/kfp/gcp.py b/sdk/python/kfp/deprecated/gcp.py similarity index 100% rename from sdk/python/kfp/gcp.py rename to sdk/python/kfp/deprecated/gcp.py diff --git a/sdk/python/kfp/notebook/__init__.py b/sdk/python/kfp/deprecated/notebook/__init__.py similarity index 100% rename from sdk/python/kfp/notebook/__init__.py rename to sdk/python/kfp/deprecated/notebook/__init__.py diff --git a/sdk/python/kfp/notebook/_magic.py b/sdk/python/kfp/deprecated/notebook/_magic.py similarity index 100% rename from sdk/python/kfp/notebook/_magic.py rename to sdk/python/kfp/deprecated/notebook/_magic.py diff --git a/sdk/python/kfp/onprem.py b/sdk/python/kfp/deprecated/onprem.py similarity index 99% rename from sdk/python/kfp/onprem.py rename to sdk/python/kfp/deprecated/onprem.py index b29a5efd1fa..5db04c9724a 100644 --- a/sdk/python/kfp/onprem.py +++ b/sdk/python/kfp/deprecated/onprem.py @@ -1,5 +1,5 @@ from typing import Dict, Optional -from kfp import dsl +from kfp.deprecated import dsl def mount_pvc(pvc_name='pipeline-claim', diff --git a/sdk/python/kfp/v2/components/structures.py b/sdk/python/kfp/v2/components/structures.py index 5edfcaf571e..fcd5e48ca64 100644 --- a/sdk/python/kfp/v2/components/structures.py +++ b/sdk/python/kfp/v2/components/structures.py @@ -18,8 +18,8 @@ import json from typing import Any, Dict, Mapping, Optional, Sequence, Union -from kfp.components import _components -from kfp.components import structures as v1_structures +from kfp.deprecated.components import _components +from kfp.deprecated.components import structures as v1_structures from kfp.v2.components import utils import pydantic import yaml diff --git a/sdk/python/kfp/v2/components/types/type_utils.py b/sdk/python/kfp/v2/components/types/type_utils.py index 06ea47e40f0..d120b54ba9e 100644 --- a/sdk/python/kfp/v2/components/types/type_utils.py +++ b/sdk/python/kfp/v2/components/types/type_utils.py @@ -18,7 +18,7 @@ import warnings from typing import Dict, List, Optional, Type, Union -from kfp.components import structures +from kfp.deprecated.components import structures from kfp.pipeline_spec import pipeline_spec_pb2 from kfp.v2.components.types import artifact_types from kfp.v2.components.types import type_annotations From 405f51cb6a5bfee1824541b4f1caef63ed7b5f53 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 9 Feb 2022 18:34:28 -0800 Subject: [PATCH 02/39] fix testsg --- sdk/python/setup.py | 24 +++++++++---------- sdk/python/tests/compiler/compiler_tests.py | 14 +++++------ .../tests/compiler/component_builder_test.py | 2 +- .../tests/compiler/container_builder_test.py | 2 +- sdk/python/tests/compiler/k8s_helper_tests.py | 2 +- sdk/python/tests/dsl/aws_extensions_tests.py | 4 ++-- .../tests/dsl/component_bridge_tests.py | 8 +++---- sdk/python/tests/dsl/component_tests.py | 12 +++++----- sdk/python/tests/dsl/container_op_tests.py | 4 ++-- sdk/python/tests/dsl/metadata_tests.py | 2 +- sdk/python/tests/dsl/ops_group_tests.py | 4 ++-- sdk/python/tests/dsl/pipeline_param_tests.py | 4 ++-- sdk/python/tests/dsl/pipeline_tests.py | 10 ++++---- sdk/python/tests/dsl/pipeline_volume_tests.py | 4 ++-- sdk/python/tests/dsl/resource_op_tests.py | 4 ++-- sdk/python/tests/dsl/test_azure_extensions.py | 4 ++-- sdk/python/tests/dsl/type_tests.py | 2 +- sdk/python/tests/dsl/volume_op_tests.py | 4 ++-- .../tests/dsl/volume_snapshotop_tests.py | 4 ++-- sdk/python/tests/local_runner_test.py | 8 +++---- 20 files changed, 61 insertions(+), 61 deletions(-) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index b5e1e8b812e..3525327ebd0 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -98,17 +98,17 @@ def find_version(*file_path_parts): install_requires=REQUIRES, extras_require=EXTRAS_REQUIRE, packages=[ - 'kfp', - 'kfp.auth', - 'kfp.cli', - 'kfp.cli.diagnose_me', - 'kfp.compiler', - 'kfp.components', - 'kfp.components.structures', - 'kfp.containers', - 'kfp.dsl', - 'kfp.dsl.extensions', - 'kfp.notebook', + 'kfp.deprecated', + 'kfp.deprecated.auth', + 'kfp.deprecated.cli', + 'kfp.deprecated.cli.diagnose_me', + 'kfp.deprecated.compiler', + 'kfp.deprecated.components', + 'kfp.deprecated.components.structures', + 'kfp.deprecated.containers', + 'kfp.deprecated.dsl', + 'kfp.deprecated.dsl.extensions', + 'kfp.deprecated.notebook', 'kfp.v2', 'kfp.v2.compiler', 'kfp.v2.components', @@ -135,7 +135,7 @@ def find_version(*file_path_parts): include_package_data=True, entry_points={ 'console_scripts': [ - 'dsl-compile = kfp.compiler.main:main', + 'dsl-compile = kfp.deprecated.compiler.main:main', 'dsl-compile-v2 = kfp.v2.compiler.main:main', 'kfp=kfp.__main__:main' ] diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py index 89fa775134b..a4922b35ce8 100644 --- a/sdk/python/tests/compiler/compiler_tests.py +++ b/sdk/python/tests/compiler/compiler_tests.py @@ -14,9 +14,9 @@ from typing import Optional import unittest -import kfp -import kfp.compiler as compiler -import kfp.dsl as dsl +import kfp.deprecated as kfp +import kfp.deprecated.compiler as compiler +import kfp.deprecated.dsl as dsl import json import os import shutil @@ -29,10 +29,10 @@ from unittest import mock from absl.testing import parameterized -from kfp.compiler import Compiler -from kfp.dsl._component import component -from kfp.dsl import ContainerOp, pipeline, PipelineParam -from kfp.dsl.types import Integer, InconsistentTypeException +from kfp.deprecated.compiler import Compiler +from kfp.deprecated.dsl._component import component +from kfp.deprecated.dsl import ContainerOp, pipeline, PipelineParam +from kfp.deprecated.dsl.types import Integer, InconsistentTypeException from kubernetes.client import V1Toleration, V1Affinity, V1NodeSelector, V1NodeSelectorRequirement, V1NodeSelectorTerm, \ V1NodeAffinity, V1PodDNSConfig, V1PodDNSConfigOption diff --git a/sdk/python/tests/compiler/component_builder_test.py b/sdk/python/tests/compiler/component_builder_test.py index b34c5fc9e0b..6d6cfa738be 100644 --- a/sdk/python/tests/compiler/component_builder_test.py +++ b/sdk/python/tests/compiler/component_builder_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.containers._component_builder import _generate_dockerfile, _dependency_to_requirements, VersionedDependency, DependencyHelper +from kfp.deprecated.containers._component_builder import _generate_dockerfile, _dependency_to_requirements, VersionedDependency, DependencyHelper import os import unittest diff --git a/sdk/python/tests/compiler/container_builder_test.py b/sdk/python/tests/compiler/container_builder_test.py index 8c0d2e49117..98c8e48a3bf 100644 --- a/sdk/python/tests/compiler/container_builder_test.py +++ b/sdk/python/tests/compiler/container_builder_test.py @@ -18,7 +18,7 @@ import yaml import tempfile from unittest import mock -from kfp.containers._component_builder import ContainerBuilder +from kfp.deprecated.containers._component_builder import ContainerBuilder GCS_BASE = 'gs://kfp-testing/' DEFAULT_IMAGE_NAME = 'gcr.io/kfp-testing/image' diff --git a/sdk/python/tests/compiler/k8s_helper_tests.py b/sdk/python/tests/compiler/k8s_helper_tests.py index 08d86c8136d..e7f6fe76769 100644 --- a/sdk/python/tests/compiler/k8s_helper_tests.py +++ b/sdk/python/tests/compiler/k8s_helper_tests.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.compiler._k8s_helper import convert_k8s_obj_to_json +from kfp.deprecated.compiler._k8s_helper import convert_k8s_obj_to_json from datetime import datetime import unittest diff --git a/sdk/python/tests/dsl/aws_extensions_tests.py b/sdk/python/tests/dsl/aws_extensions_tests.py index 59995934727..eaaf9208451 100644 --- a/sdk/python/tests/dsl/aws_extensions_tests.py +++ b/sdk/python/tests/dsl/aws_extensions_tests.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.dsl import ContainerOp -from kfp.aws import use_aws_secret +from kfp.deprecated.dsl import ContainerOp +from kfp.deprecated.aws import use_aws_secret import unittest import inspect diff --git a/sdk/python/tests/dsl/component_bridge_tests.py b/sdk/python/tests/dsl/component_bridge_tests.py index 0eaa69a1d72..0426a87b8a9 100644 --- a/sdk/python/tests/dsl/component_bridge_tests.py +++ b/sdk/python/tests/dsl/component_bridge_tests.py @@ -16,11 +16,11 @@ import textwrap import unittest import warnings -import kfp +import kfp.deprecated as kfp from pathlib import Path -from kfp.components import load_component_from_text, create_component_from_func -from kfp.dsl.types import InconsistentTypeException -from kfp.dsl import PipelineParam +from kfp.deprecated.components import load_component_from_text, create_component_from_func +from kfp.deprecated.dsl.types import InconsistentTypeException +from kfp.deprecated.dsl import PipelineParam class TestComponentBridge(unittest.TestCase): diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index 7a893abe051..cb3fb865f5e 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -16,12 +16,12 @@ from typing import List, Optional import unittest -import kfp -import kfp.dsl as dsl -from kfp.dsl import component, graph_component -from kfp.dsl.types import Integer, GCSPath, InconsistentTypeException -from kfp.dsl import ContainerOp, Pipeline, PipelineParam -from kfp.components.structures import ComponentSpec, InputSpec, OutputSpec +import kfp.deprecated as kfp +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import component, graph_component +from kfp.deprecated.dsl.types import Integer, GCSPath, InconsistentTypeException +from kfp.deprecated.dsl import ContainerOp, Pipeline, PipelineParam +from kfp.deprecated.components.structures import ComponentSpec, InputSpec, OutputSpec class TestPythonComponent(unittest.TestCase): diff --git a/sdk/python/tests/dsl/container_op_tests.py b/sdk/python/tests/dsl/container_op_tests.py index 24a83efa081..1c4280b800d 100644 --- a/sdk/python/tests/dsl/container_op_tests.py +++ b/sdk/python/tests/dsl/container_op_tests.py @@ -15,8 +15,8 @@ import unittest from kubernetes.client.models import V1EnvVar, V1VolumeMount -import kfp -from kfp.dsl import ContainerOp, UserContainer, Sidecar, PipelineVolume +import kfp.deprecated as kfp +from kfp.deprecated.dsl import ContainerOp, UserContainer, Sidecar, PipelineVolume class TestContainerOp(unittest.TestCase): diff --git a/sdk/python/tests/dsl/metadata_tests.py b/sdk/python/tests/dsl/metadata_tests.py index ce7566e87a6..76c09c7b6b1 100644 --- a/sdk/python/tests/dsl/metadata_tests.py +++ b/sdk/python/tests/dsl/metadata_tests.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.components.structures import ComponentSpec, InputSpec, OutputSpec +from kfp.deprecated.components.structures import ComponentSpec, InputSpec, OutputSpec import unittest diff --git a/sdk/python/tests/dsl/ops_group_tests.py b/sdk/python/tests/dsl/ops_group_tests.py index 90ebae1ba8f..f64fb6cbe7b 100644 --- a/sdk/python/tests/dsl/ops_group_tests.py +++ b/sdk/python/tests/dsl/ops_group_tests.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.dsl import Pipeline, PipelineParam, ContainerOp, ExitHandler, OpsGroup, Condition +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import Pipeline, PipelineParam, ContainerOp, ExitHandler, OpsGroup, Condition import unittest diff --git a/sdk/python/tests/dsl/pipeline_param_tests.py b/sdk/python/tests/dsl/pipeline_param_tests.py index 08aefaa67fe..e3b8756e2b4 100644 --- a/sdk/python/tests/dsl/pipeline_param_tests.py +++ b/sdk/python/tests/dsl/pipeline_param_tests.py @@ -13,8 +13,8 @@ # limitations under the License. from kubernetes.client.models import V1ConfigMap, V1Container, V1EnvVar -from kfp.dsl import PipelineParam -from kfp.dsl._pipeline_param import _extract_pipelineparams, extract_pipelineparams_from_any +from kfp.deprecated.dsl import PipelineParam +from kfp.deprecated.dsl._pipeline_param import _extract_pipelineparams, extract_pipelineparams_from_any import unittest diff --git a/sdk/python/tests/dsl/pipeline_tests.py b/sdk/python/tests/dsl/pipeline_tests.py index 561fddaa4fa..4d3b9d99bc6 100644 --- a/sdk/python/tests/dsl/pipeline_tests.py +++ b/sdk/python/tests/dsl/pipeline_tests.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.dsl import Pipeline, PipelineParam, ContainerOp, pipeline -from kfp.dsl._metadata import _extract_pipeline_metadata -from kfp.dsl.types import GCSPath, Integer -from kfp.components.structures import ComponentSpec, InputSpec +import kfp.deprecated as kfp +from kfp.deprecated.dsl import Pipeline, PipelineParam, ContainerOp, pipeline +from kfp.deprecated.dsl._metadata import _extract_pipeline_metadata +from kfp.deprecated.dsl.types import GCSPath, Integer +from kfp.deprecated.components.structures import ComponentSpec, InputSpec import unittest diff --git a/sdk/python/tests/dsl/pipeline_volume_tests.py b/sdk/python/tests/dsl/pipeline_volume_tests.py index 15f32707034..1669970eacb 100644 --- a/sdk/python/tests/dsl/pipeline_volume_tests.py +++ b/sdk/python/tests/dsl/pipeline_volume_tests.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.dsl import VolumeOp, ContainerOp, PipelineVolume +import kfp.deprecated as kfp +from kfp.deprecated.dsl import VolumeOp, ContainerOp, PipelineVolume import unittest diff --git a/sdk/python/tests/dsl/resource_op_tests.py b/sdk/python/tests/dsl/resource_op_tests.py index 2e430874f5d..fceaad06798 100644 --- a/sdk/python/tests/dsl/resource_op_tests.py +++ b/sdk/python/tests/dsl/resource_op_tests.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.dsl import PipelineParam, ResourceOp +import kfp.deprecated as kfp +from kfp.deprecated.dsl import PipelineParam, ResourceOp from kubernetes import client as k8s_client import unittest diff --git a/sdk/python/tests/dsl/test_azure_extensions.py b/sdk/python/tests/dsl/test_azure_extensions.py index f56e1cd013e..6ced0e7910c 100644 --- a/sdk/python/tests/dsl/test_azure_extensions.py +++ b/sdk/python/tests/dsl/test_azure_extensions.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.dsl import ContainerOp -from kfp.azure import use_azure_secret +from kfp.deprecated.dsl import ContainerOp +from kfp.deprecated.azure import use_azure_secret import unittest import inspect diff --git a/sdk/python/tests/dsl/type_tests.py b/sdk/python/tests/dsl/type_tests.py index f0b9ecfa1cd..117c6a40d52 100644 --- a/sdk/python/tests/dsl/type_tests.py +++ b/sdk/python/tests/dsl/type_tests.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.dsl.types import check_types, GCSPath +from kfp.deprecated.dsl.types import check_types, GCSPath import unittest diff --git a/sdk/python/tests/dsl/volume_op_tests.py b/sdk/python/tests/dsl/volume_op_tests.py index 9d12d44474c..be3c243e7c4 100644 --- a/sdk/python/tests/dsl/volume_op_tests.py +++ b/sdk/python/tests/dsl/volume_op_tests.py @@ -15,8 +15,8 @@ from kubernetes.client.models import (V1Volume, V1PersistentVolumeClaimVolumeSource) -import kfp -from kfp.dsl import PipelineParam, VolumeOp, PipelineVolume +import kfp.deprecated as kfp +from kfp.deprecated.dsl import PipelineParam, VolumeOp, PipelineVolume import unittest diff --git a/sdk/python/tests/dsl/volume_snapshotop_tests.py b/sdk/python/tests/dsl/volume_snapshotop_tests.py index 8be8bfd6108..ba4972653ac 100644 --- a/sdk/python/tests/dsl/volume_snapshotop_tests.py +++ b/sdk/python/tests/dsl/volume_snapshotop_tests.py @@ -13,8 +13,8 @@ # limitations under the License. from kubernetes import client as k8s_client -import kfp -from kfp.dsl import PipelineParam, VolumeOp, VolumeSnapshotOp +import kfp.deprecated as kfp +from kfp.deprecated.dsl import PipelineParam, VolumeOp, VolumeSnapshotOp import unittest diff --git a/sdk/python/tests/local_runner_test.py b/sdk/python/tests/local_runner_test.py index 51126cafd89..181da750548 100644 --- a/sdk/python/tests/local_runner_test.py +++ b/sdk/python/tests/local_runner_test.py @@ -15,8 +15,8 @@ import unittest from typing import Callable -import kfp -from kfp import LocalClient, run_pipeline_func_locally +import kfp.deprecated as kfp +from kfp.deprecated import LocalClient, run_pipeline_func_locally InputPath = kfp.components.InputPath() OutputPath = kfp.components.OutputPath() @@ -226,7 +226,7 @@ def check_option(dst: OutputPath): def _pipeline(): check_option() - + run_result = run_pipeline_func_locally( _pipeline, {}, @@ -235,7 +235,7 @@ def _pipeline(): ) assert run_result.success output_file_path = run_result.get_output_file("check-option") - + with open(output_file_path, "r") as f: line = f.readline() assert "bar" in line \ No newline at end of file From 5ee1fe9fd8b54ec4efa1d93c54051d9c01236b3a Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 9 Feb 2022 18:34:28 -0800 Subject: [PATCH 03/39] fix testsg --- sdk/python/kfp/__init__.py | 13 ++++++++++ sdk/python/setup.py | 24 +++++++++---------- sdk/python/tests/compiler/compiler_tests.py | 14 +++++------ .../tests/compiler/component_builder_test.py | 2 +- .../tests/compiler/container_builder_test.py | 2 +- sdk/python/tests/compiler/k8s_helper_tests.py | 2 +- sdk/python/tests/dsl/aws_extensions_tests.py | 4 ++-- .../tests/dsl/component_bridge_tests.py | 8 +++---- sdk/python/tests/dsl/component_tests.py | 12 +++++----- sdk/python/tests/dsl/container_op_tests.py | 4 ++-- sdk/python/tests/dsl/metadata_tests.py | 2 +- sdk/python/tests/dsl/ops_group_tests.py | 4 ++-- sdk/python/tests/dsl/pipeline_param_tests.py | 4 ++-- sdk/python/tests/dsl/pipeline_tests.py | 10 ++++---- sdk/python/tests/dsl/pipeline_volume_tests.py | 4 ++-- sdk/python/tests/dsl/resource_op_tests.py | 4 ++-- sdk/python/tests/dsl/test_azure_extensions.py | 4 ++-- sdk/python/tests/dsl/type_tests.py | 2 +- sdk/python/tests/dsl/volume_op_tests.py | 4 ++-- .../tests/dsl/volume_snapshotop_tests.py | 4 ++-- sdk/python/tests/local_runner_test.py | 8 +++---- 21 files changed, 74 insertions(+), 61 deletions(-) create mode 100644 sdk/python/kfp/__init__.py diff --git a/sdk/python/kfp/__init__.py b/sdk/python/kfp/__init__.py new file mode 100644 index 00000000000..c55179a49ed --- /dev/null +++ b/sdk/python/kfp/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/sdk/python/setup.py b/sdk/python/setup.py index b5e1e8b812e..3525327ebd0 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -98,17 +98,17 @@ def find_version(*file_path_parts): install_requires=REQUIRES, extras_require=EXTRAS_REQUIRE, packages=[ - 'kfp', - 'kfp.auth', - 'kfp.cli', - 'kfp.cli.diagnose_me', - 'kfp.compiler', - 'kfp.components', - 'kfp.components.structures', - 'kfp.containers', - 'kfp.dsl', - 'kfp.dsl.extensions', - 'kfp.notebook', + 'kfp.deprecated', + 'kfp.deprecated.auth', + 'kfp.deprecated.cli', + 'kfp.deprecated.cli.diagnose_me', + 'kfp.deprecated.compiler', + 'kfp.deprecated.components', + 'kfp.deprecated.components.structures', + 'kfp.deprecated.containers', + 'kfp.deprecated.dsl', + 'kfp.deprecated.dsl.extensions', + 'kfp.deprecated.notebook', 'kfp.v2', 'kfp.v2.compiler', 'kfp.v2.components', @@ -135,7 +135,7 @@ def find_version(*file_path_parts): include_package_data=True, entry_points={ 'console_scripts': [ - 'dsl-compile = kfp.compiler.main:main', + 'dsl-compile = kfp.deprecated.compiler.main:main', 'dsl-compile-v2 = kfp.v2.compiler.main:main', 'kfp=kfp.__main__:main' ] diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py index 89fa775134b..a4922b35ce8 100644 --- a/sdk/python/tests/compiler/compiler_tests.py +++ b/sdk/python/tests/compiler/compiler_tests.py @@ -14,9 +14,9 @@ from typing import Optional import unittest -import kfp -import kfp.compiler as compiler -import kfp.dsl as dsl +import kfp.deprecated as kfp +import kfp.deprecated.compiler as compiler +import kfp.deprecated.dsl as dsl import json import os import shutil @@ -29,10 +29,10 @@ from unittest import mock from absl.testing import parameterized -from kfp.compiler import Compiler -from kfp.dsl._component import component -from kfp.dsl import ContainerOp, pipeline, PipelineParam -from kfp.dsl.types import Integer, InconsistentTypeException +from kfp.deprecated.compiler import Compiler +from kfp.deprecated.dsl._component import component +from kfp.deprecated.dsl import ContainerOp, pipeline, PipelineParam +from kfp.deprecated.dsl.types import Integer, InconsistentTypeException from kubernetes.client import V1Toleration, V1Affinity, V1NodeSelector, V1NodeSelectorRequirement, V1NodeSelectorTerm, \ V1NodeAffinity, V1PodDNSConfig, V1PodDNSConfigOption diff --git a/sdk/python/tests/compiler/component_builder_test.py b/sdk/python/tests/compiler/component_builder_test.py index b34c5fc9e0b..6d6cfa738be 100644 --- a/sdk/python/tests/compiler/component_builder_test.py +++ b/sdk/python/tests/compiler/component_builder_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.containers._component_builder import _generate_dockerfile, _dependency_to_requirements, VersionedDependency, DependencyHelper +from kfp.deprecated.containers._component_builder import _generate_dockerfile, _dependency_to_requirements, VersionedDependency, DependencyHelper import os import unittest diff --git a/sdk/python/tests/compiler/container_builder_test.py b/sdk/python/tests/compiler/container_builder_test.py index 8c0d2e49117..98c8e48a3bf 100644 --- a/sdk/python/tests/compiler/container_builder_test.py +++ b/sdk/python/tests/compiler/container_builder_test.py @@ -18,7 +18,7 @@ import yaml import tempfile from unittest import mock -from kfp.containers._component_builder import ContainerBuilder +from kfp.deprecated.containers._component_builder import ContainerBuilder GCS_BASE = 'gs://kfp-testing/' DEFAULT_IMAGE_NAME = 'gcr.io/kfp-testing/image' diff --git a/sdk/python/tests/compiler/k8s_helper_tests.py b/sdk/python/tests/compiler/k8s_helper_tests.py index 08d86c8136d..e7f6fe76769 100644 --- a/sdk/python/tests/compiler/k8s_helper_tests.py +++ b/sdk/python/tests/compiler/k8s_helper_tests.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.compiler._k8s_helper import convert_k8s_obj_to_json +from kfp.deprecated.compiler._k8s_helper import convert_k8s_obj_to_json from datetime import datetime import unittest diff --git a/sdk/python/tests/dsl/aws_extensions_tests.py b/sdk/python/tests/dsl/aws_extensions_tests.py index 59995934727..eaaf9208451 100644 --- a/sdk/python/tests/dsl/aws_extensions_tests.py +++ b/sdk/python/tests/dsl/aws_extensions_tests.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.dsl import ContainerOp -from kfp.aws import use_aws_secret +from kfp.deprecated.dsl import ContainerOp +from kfp.deprecated.aws import use_aws_secret import unittest import inspect diff --git a/sdk/python/tests/dsl/component_bridge_tests.py b/sdk/python/tests/dsl/component_bridge_tests.py index 0eaa69a1d72..0426a87b8a9 100644 --- a/sdk/python/tests/dsl/component_bridge_tests.py +++ b/sdk/python/tests/dsl/component_bridge_tests.py @@ -16,11 +16,11 @@ import textwrap import unittest import warnings -import kfp +import kfp.deprecated as kfp from pathlib import Path -from kfp.components import load_component_from_text, create_component_from_func -from kfp.dsl.types import InconsistentTypeException -from kfp.dsl import PipelineParam +from kfp.deprecated.components import load_component_from_text, create_component_from_func +from kfp.deprecated.dsl.types import InconsistentTypeException +from kfp.deprecated.dsl import PipelineParam class TestComponentBridge(unittest.TestCase): diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index 7a893abe051..cb3fb865f5e 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -16,12 +16,12 @@ from typing import List, Optional import unittest -import kfp -import kfp.dsl as dsl -from kfp.dsl import component, graph_component -from kfp.dsl.types import Integer, GCSPath, InconsistentTypeException -from kfp.dsl import ContainerOp, Pipeline, PipelineParam -from kfp.components.structures import ComponentSpec, InputSpec, OutputSpec +import kfp.deprecated as kfp +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import component, graph_component +from kfp.deprecated.dsl.types import Integer, GCSPath, InconsistentTypeException +from kfp.deprecated.dsl import ContainerOp, Pipeline, PipelineParam +from kfp.deprecated.components.structures import ComponentSpec, InputSpec, OutputSpec class TestPythonComponent(unittest.TestCase): diff --git a/sdk/python/tests/dsl/container_op_tests.py b/sdk/python/tests/dsl/container_op_tests.py index 24a83efa081..1c4280b800d 100644 --- a/sdk/python/tests/dsl/container_op_tests.py +++ b/sdk/python/tests/dsl/container_op_tests.py @@ -15,8 +15,8 @@ import unittest from kubernetes.client.models import V1EnvVar, V1VolumeMount -import kfp -from kfp.dsl import ContainerOp, UserContainer, Sidecar, PipelineVolume +import kfp.deprecated as kfp +from kfp.deprecated.dsl import ContainerOp, UserContainer, Sidecar, PipelineVolume class TestContainerOp(unittest.TestCase): diff --git a/sdk/python/tests/dsl/metadata_tests.py b/sdk/python/tests/dsl/metadata_tests.py index ce7566e87a6..76c09c7b6b1 100644 --- a/sdk/python/tests/dsl/metadata_tests.py +++ b/sdk/python/tests/dsl/metadata_tests.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.components.structures import ComponentSpec, InputSpec, OutputSpec +from kfp.deprecated.components.structures import ComponentSpec, InputSpec, OutputSpec import unittest diff --git a/sdk/python/tests/dsl/ops_group_tests.py b/sdk/python/tests/dsl/ops_group_tests.py index 90ebae1ba8f..f64fb6cbe7b 100644 --- a/sdk/python/tests/dsl/ops_group_tests.py +++ b/sdk/python/tests/dsl/ops_group_tests.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.dsl import Pipeline, PipelineParam, ContainerOp, ExitHandler, OpsGroup, Condition +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import Pipeline, PipelineParam, ContainerOp, ExitHandler, OpsGroup, Condition import unittest diff --git a/sdk/python/tests/dsl/pipeline_param_tests.py b/sdk/python/tests/dsl/pipeline_param_tests.py index 08aefaa67fe..e3b8756e2b4 100644 --- a/sdk/python/tests/dsl/pipeline_param_tests.py +++ b/sdk/python/tests/dsl/pipeline_param_tests.py @@ -13,8 +13,8 @@ # limitations under the License. from kubernetes.client.models import V1ConfigMap, V1Container, V1EnvVar -from kfp.dsl import PipelineParam -from kfp.dsl._pipeline_param import _extract_pipelineparams, extract_pipelineparams_from_any +from kfp.deprecated.dsl import PipelineParam +from kfp.deprecated.dsl._pipeline_param import _extract_pipelineparams, extract_pipelineparams_from_any import unittest diff --git a/sdk/python/tests/dsl/pipeline_tests.py b/sdk/python/tests/dsl/pipeline_tests.py index 561fddaa4fa..4d3b9d99bc6 100644 --- a/sdk/python/tests/dsl/pipeline_tests.py +++ b/sdk/python/tests/dsl/pipeline_tests.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.dsl import Pipeline, PipelineParam, ContainerOp, pipeline -from kfp.dsl._metadata import _extract_pipeline_metadata -from kfp.dsl.types import GCSPath, Integer -from kfp.components.structures import ComponentSpec, InputSpec +import kfp.deprecated as kfp +from kfp.deprecated.dsl import Pipeline, PipelineParam, ContainerOp, pipeline +from kfp.deprecated.dsl._metadata import _extract_pipeline_metadata +from kfp.deprecated.dsl.types import GCSPath, Integer +from kfp.deprecated.components.structures import ComponentSpec, InputSpec import unittest diff --git a/sdk/python/tests/dsl/pipeline_volume_tests.py b/sdk/python/tests/dsl/pipeline_volume_tests.py index 15f32707034..1669970eacb 100644 --- a/sdk/python/tests/dsl/pipeline_volume_tests.py +++ b/sdk/python/tests/dsl/pipeline_volume_tests.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.dsl import VolumeOp, ContainerOp, PipelineVolume +import kfp.deprecated as kfp +from kfp.deprecated.dsl import VolumeOp, ContainerOp, PipelineVolume import unittest diff --git a/sdk/python/tests/dsl/resource_op_tests.py b/sdk/python/tests/dsl/resource_op_tests.py index 2e430874f5d..fceaad06798 100644 --- a/sdk/python/tests/dsl/resource_op_tests.py +++ b/sdk/python/tests/dsl/resource_op_tests.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.dsl import PipelineParam, ResourceOp +import kfp.deprecated as kfp +from kfp.deprecated.dsl import PipelineParam, ResourceOp from kubernetes import client as k8s_client import unittest diff --git a/sdk/python/tests/dsl/test_azure_extensions.py b/sdk/python/tests/dsl/test_azure_extensions.py index f56e1cd013e..6ced0e7910c 100644 --- a/sdk/python/tests/dsl/test_azure_extensions.py +++ b/sdk/python/tests/dsl/test_azure_extensions.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.dsl import ContainerOp -from kfp.azure import use_azure_secret +from kfp.deprecated.dsl import ContainerOp +from kfp.deprecated.azure import use_azure_secret import unittest import inspect diff --git a/sdk/python/tests/dsl/type_tests.py b/sdk/python/tests/dsl/type_tests.py index f0b9ecfa1cd..117c6a40d52 100644 --- a/sdk/python/tests/dsl/type_tests.py +++ b/sdk/python/tests/dsl/type_tests.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.dsl.types import check_types, GCSPath +from kfp.deprecated.dsl.types import check_types, GCSPath import unittest diff --git a/sdk/python/tests/dsl/volume_op_tests.py b/sdk/python/tests/dsl/volume_op_tests.py index 9d12d44474c..be3c243e7c4 100644 --- a/sdk/python/tests/dsl/volume_op_tests.py +++ b/sdk/python/tests/dsl/volume_op_tests.py @@ -15,8 +15,8 @@ from kubernetes.client.models import (V1Volume, V1PersistentVolumeClaimVolumeSource) -import kfp -from kfp.dsl import PipelineParam, VolumeOp, PipelineVolume +import kfp.deprecated as kfp +from kfp.deprecated.dsl import PipelineParam, VolumeOp, PipelineVolume import unittest diff --git a/sdk/python/tests/dsl/volume_snapshotop_tests.py b/sdk/python/tests/dsl/volume_snapshotop_tests.py index 8be8bfd6108..ba4972653ac 100644 --- a/sdk/python/tests/dsl/volume_snapshotop_tests.py +++ b/sdk/python/tests/dsl/volume_snapshotop_tests.py @@ -13,8 +13,8 @@ # limitations under the License. from kubernetes import client as k8s_client -import kfp -from kfp.dsl import PipelineParam, VolumeOp, VolumeSnapshotOp +import kfp.deprecated as kfp +from kfp.deprecated.dsl import PipelineParam, VolumeOp, VolumeSnapshotOp import unittest diff --git a/sdk/python/tests/local_runner_test.py b/sdk/python/tests/local_runner_test.py index 51126cafd89..181da750548 100644 --- a/sdk/python/tests/local_runner_test.py +++ b/sdk/python/tests/local_runner_test.py @@ -15,8 +15,8 @@ import unittest from typing import Callable -import kfp -from kfp import LocalClient, run_pipeline_func_locally +import kfp.deprecated as kfp +from kfp.deprecated import LocalClient, run_pipeline_func_locally InputPath = kfp.components.InputPath() OutputPath = kfp.components.OutputPath() @@ -226,7 +226,7 @@ def check_option(dst: OutputPath): def _pipeline(): check_option() - + run_result = run_pipeline_func_locally( _pipeline, {}, @@ -235,7 +235,7 @@ def _pipeline(): ) assert run_result.success output_file_path = run_result.get_output_file("check-option") - + with open(output_file_path, "r") as f: line = f.readline() assert "bar" in line \ No newline at end of file From e401eb080688497e4e2addd851ca3c111436d08c Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 9 Feb 2022 18:49:31 -0800 Subject: [PATCH 04/39] fix setup.py --- sdk/python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 3525327ebd0..f6ef21e6581 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -98,6 +98,7 @@ def find_version(*file_path_parts): install_requires=REQUIRES, extras_require=EXTRAS_REQUIRE, packages=[ + 'kfp', 'kfp.deprecated', 'kfp.deprecated.auth', 'kfp.deprecated.cli', From 9e3490df8fdb92cecd10888f22b4cf3b17762d62 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 9 Feb 2022 19:25:56 -0800 Subject: [PATCH 05/39] fix test --- backend/src/v2/test/sample_test.py | 2 +- sdk/python/kfp/deprecated/_client.py | 2 +- sdk/python/kfp/deprecated/cli/components.py | 2 +- sdk/python/kfp/deprecated/cli/components_test.py | 2 +- .../kfp/deprecated/cli/diagnose_me/dev_env_test.py | 4 ++-- sdk/python/kfp/deprecated/dsl/_component_bridge.py | 2 +- sdk/python/kfp/deprecated/dsl/_container_op.py | 2 +- sdk/python/kfp/deprecated/dsl/types.py | 2 +- sdk/python/kfp/v2/compiler/compiler.py | 2 +- sdk/python/kfp/v2/compiler/main.py | 2 +- sdk/python/tests/compiler/compiler_tests.py | 2 +- sdk/python/tests/compiler/testdata/add_pod_env.py | 10 +++++----- .../compiler/testdata/artifact_passing_using_volume.py | 6 +++--- sdk/python/tests/compiler/testdata/basic.py | 4 ++-- .../tests/compiler/testdata/basic_no_decorator.py | 4 ++-- sdk/python/tests/compiler/testdata/coin.py | 2 +- sdk/python/tests/compiler/testdata/compose.py | 2 +- sdk/python/tests/compiler/testdata/default_value.py | 2 +- .../compiler/testdata/input_artifact_raw_value.py | 4 ++-- .../compiler/testdata/loop_over_lightweight_output.py | 6 +++--- sdk/python/tests/compiler/testdata/opsgroups.py | 4 ++-- .../testdata/parallelfor_item_argument_resolving.py | 6 +++--- .../parallelfor_pipeline_param_in_items_resolving.py | 6 +++--- .../tests/compiler/testdata/param_op_transform.py | 4 ++-- .../tests/compiler/testdata/param_substitutions.py | 4 ++-- sdk/python/tests/compiler/testdata/pipelineparams.py | 2 +- .../tests/compiler/testdata/recursive_do_while.py | 6 +++--- sdk/python/tests/compiler/testdata/recursive_while.py | 4 ++-- sdk/python/tests/compiler/testdata/resourceop_basic.py | 4 ++-- sdk/python/tests/compiler/testdata/sidecar.py | 2 +- .../testdata/testpackage/mypipeline/compose.py | 2 +- sdk/python/tests/compiler/testdata/timeout.py | 4 ++-- sdk/python/tests/compiler/testdata/two_step.py | 4 ++-- sdk/python/tests/compiler/testdata/uri_artifacts.py | 6 +++--- sdk/python/tests/compiler/testdata/volume.py | 2 +- .../compiler/testdata/volume_snapshotop_rokurl.py | 4 ++-- .../compiler/testdata/volume_snapshotop_sequential.py | 4 ++-- sdk/python/tests/compiler/testdata/volumeop_basic.py | 4 ++-- sdk/python/tests/compiler/testdata/volumeop_dag.py | 4 ++-- .../tests/compiler/testdata/volumeop_parallel.py | 4 ++-- .../tests/compiler/testdata/volumeop_sequential.py | 4 ++-- sdk/python/tests/compiler/testdata/withitem_basic.py | 8 ++++---- sdk/python/tests/compiler/testdata/withitem_nested.py | 8 ++++---- sdk/python/tests/compiler/testdata/withparam_global.py | 8 ++++---- .../tests/compiler/testdata/withparam_global_dict.py | 8 ++++---- sdk/python/tests/compiler/testdata/withparam_output.py | 8 ++++---- .../tests/compiler/testdata/withparam_output_dict.py | 8 ++++---- sdk/python/tests/dsl/component_bridge_tests.py | 4 ++-- sdk/python/tests/dsl/component_tests.py | 4 ++-- sdk/python/tests/dsl/extensions/test_kubernetes.py | 4 ++-- test/kfp-functional-test/run_kfp_functional_test.py | 4 ++-- test/sample-test/check_notebook_results.py | 4 ++-- test/sample-test/run_resnet_cmle_test.py | 2 +- test/sample-test/run_sample_test.py | 4 ++-- test/sample-test/sample_test_launcher.py | 2 +- 55 files changed, 112 insertions(+), 112 deletions(-) diff --git a/backend/src/v2/test/sample_test.py b/backend/src/v2/test/sample_test.py index 1ef900b1fad..6cb4c12a309 100644 --- a/backend/src/v2/test/sample_test.py +++ b/backend/src/v2/test/sample_test.py @@ -18,7 +18,7 @@ import json import yaml from kubernetes import client as k8s_client -import kfp +import kfp.deprecated as kfp download_gcs_tgz = kfp.components.load_component_from_file( 'components/download_gcs_tgz.yaml') diff --git a/sdk/python/kfp/deprecated/_client.py b/sdk/python/kfp/deprecated/_client.py index cd79ce2f691..e64d44a6aba 100644 --- a/sdk/python/kfp/deprecated/_client.py +++ b/sdk/python/kfp/deprecated/_client.py @@ -364,7 +364,7 @@ def _get_config_with_default_credentials(self, config): # implement more and more credentials, we can have some heuristic and # choose from a number of options. # See https://github.com/kubeflow/pipelines/pull/5287#issuecomment-805654121 - from kfp import auth + from kfp.deprecated import auth credentials = auth.ServiceAccountTokenVolumeCredentials() config_copy = copy.deepcopy(config) diff --git a/sdk/python/kfp/deprecated/cli/components.py b/sdk/python/kfp/deprecated/cli/components.py index fc40235c463..170dcac68be 100644 --- a/sdk/python/kfp/deprecated/cli/components.py +++ b/sdk/python/kfp/deprecated/cli/components.py @@ -28,7 +28,7 @@ import typer -import kfp.deprecated +import kfp.deprecated as kfp from kfp.v2.components import component_factory, kfp_config, utils _REQUIREMENTS_TXT = 'requirements.txt' diff --git a/sdk/python/kfp/deprecated/cli/components_test.py b/sdk/python/kfp/deprecated/cli/components_test.py index fca072ea3f6..5b1881f3a5b 100644 --- a/sdk/python/kfp/deprecated/cli/components_test.py +++ b/sdk/python/kfp/deprecated/cli/components_test.py @@ -27,7 +27,7 @@ # So we patch it before importing kfp.cli.components. if importlib.util.find_spec('docker') is None: sys.modules['docker'] = mock.Mock() -from kfp.cli import components +from kfp.deprecated.cli import components _COMPONENT_TEMPLATE = ''' from kfp.v2.dsl import * diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py b/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py index aa62b92e5cf..66d304d3d95 100644 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py +++ b/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py @@ -17,8 +17,8 @@ from typing import Text import unittest from unittest import mock -from kfp.cli.diagnose_me import dev_env -from kfp.cli.diagnose_me import utility +from kfp.deprecated.cli.diagnose_me import dev_env +from kfp.deprecated.cli.diagnose_me import utility class DevEnvTest(unittest.TestCase): diff --git a/sdk/python/kfp/deprecated/dsl/_component_bridge.py b/sdk/python/kfp/deprecated/dsl/_component_bridge.py index 6c85029e4f7..eda811b3276 100644 --- a/sdk/python/kfp/deprecated/dsl/_component_bridge.py +++ b/sdk/python/kfp/deprecated/dsl/_component_bridge.py @@ -19,7 +19,7 @@ import pathlib from typing import Any, Mapping, Optional -import kfp.deprecated +import kfp.deprecated as kfp from kfp.deprecated.components import _structures, _data_passing from kfp.deprecated.components import _components from kfp.deprecated.components import _naming diff --git a/sdk/python/kfp/deprecated/dsl/_container_op.py b/sdk/python/kfp/deprecated/dsl/_container_op.py index 6acb1605b64..83f28cf7483 100644 --- a/sdk/python/kfp/deprecated/dsl/_container_op.py +++ b/sdk/python/kfp/deprecated/dsl/_container_op.py @@ -17,7 +17,7 @@ from typing import (Any, Callable, Dict, List, Optional, Sequence, Tuple, TypeVar, Union) -import kfp.deprecated +import kfp.deprecated as kfp from kfp.deprecated.components import _components, _structures from kfp.deprecated.dsl import _pipeline_param, dsl_utils from kfp.pipeline_spec import pipeline_spec_pb2 diff --git a/sdk/python/kfp/deprecated/dsl/types.py b/sdk/python/kfp/deprecated/dsl/types.py index e4940142cfa..d26224bafb6 100644 --- a/sdk/python/kfp/deprecated/dsl/types.py +++ b/sdk/python/kfp/deprecated/dsl/types.py @@ -157,7 +157,7 @@ def verify_type_compatibility(given_type: TypeSpecType, error_text = error_message_prefix + ( 'Argument type "{}" is incompatible with the input type "{}"' ).format(str(given_type), str(expected_type)) - import kfp.deprecated + import kfp.deprecated as kfp if kfp.TYPE_CHECK: raise InconsistentTypeException(error_text) else: diff --git a/sdk/python/kfp/v2/compiler/compiler.py b/sdk/python/kfp/v2/compiler/compiler.py index 8c6ba84728d..7ee2a1ac694 100644 --- a/sdk/python/kfp/v2/compiler/compiler.py +++ b/sdk/python/kfp/v2/compiler/compiler.py @@ -25,7 +25,7 @@ from typing import (Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Union) -import kfp +import kfp.deprecated as kfp from google.protobuf import json_format from kfp.pipeline_spec import pipeline_spec_pb2 from kfp.v2 import dsl diff --git a/sdk/python/kfp/v2/compiler/main.py b/sdk/python/kfp/v2/compiler/main.py index 207358dab3d..4521d907804 100644 --- a/sdk/python/kfp/v2/compiler/main.py +++ b/sdk/python/kfp/v2/compiler/main.py @@ -19,7 +19,7 @@ import sys from typing import Any, Callable, List, Mapping, Optional -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl from kfp.v2 import compiler from kfp.v2.components import pipeline_context diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py index a4922b35ce8..e2909164c35 100644 --- a/sdk/python/tests/compiler/compiler_tests.py +++ b/sdk/python/tests/compiler/compiler_tests.py @@ -912,7 +912,7 @@ def test_tolerations(self): def test_set_display_name(self): """Test a pipeline with a customized task names.""" - import kfp + import kfp.deprecated as kfp op1 = kfp.components.load_component_from_text(''' name: Component name implementation: diff --git a/sdk/python/tests/compiler/testdata/add_pod_env.py b/sdk/python/tests/compiler/testdata/add_pod_env.py index e36638ee499..f33824e7f0e 100644 --- a/sdk/python/tests/compiler/testdata/add_pod_env.py +++ b/sdk/python/tests/compiler/testdata/add_pod_env.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -import kfp.dsl as dsl +import kfp.deprecated as kfp -@dsl.pipeline(name='Test adding pod env', description='Test adding pod env') +@kfp.dsl.pipeline(name='Test adding pod env', description='Test adding pod env') def test_add_pod_env(): - op = dsl.ContainerOp( + op = kfp.dsl.ContainerOp( name='echo', image='library/bash', command=['sh', '-c'], @@ -26,4 +25,5 @@ def test_add_pod_env(): if __name__ == '__main__': - kfp.compiler.Compiler().compile(test_add_pod_env, __file__ + '.yaml') + import kfp.deprecated.compiler as compiler + compiler.Compiler().compile(test_add_pod_env, __file__ + '.yaml') diff --git a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.py b/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.py index 4d170471d59..3f250488637 100644 --- a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.py +++ b/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.py @@ -1,7 +1,7 @@ from pathlib import Path -import kfp -from kfp.components import load_component_from_file, create_component_from_func +import kfp.deprecated as kfp +from kfp.deprecated.components import load_component_from_file, create_component_from_func from typing import NamedTuple test_data_dir = Path(__file__).parent / 'test_data' @@ -55,7 +55,7 @@ def artifact_passing_pipeline(): from kubernetes.client.models import V1Volume, V1PersistentVolumeClaimVolumeSource -from kfp.dsl import data_passing_methods +from kfp.deprecated.dsl import data_passing_methods volume_based_data_passing_method = data_passing_methods.KubernetesVolume( volume=V1Volume( diff --git a/sdk/python/tests/compiler/testdata/basic.py b/sdk/python/tests/compiler/testdata/basic.py index 9570378ebce..8997bd30548 100644 --- a/sdk/python/tests/compiler/testdata/basic.py +++ b/sdk/python/tests/compiler/testdata/basic.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -import kfp.gcp as gcp +import kfp.deprecated.dsl as dsl +import kfp.deprecated.gcp as gcp class GetFrequentWordOp(dsl.ContainerOp): diff --git a/sdk/python/tests/compiler/testdata/basic_no_decorator.py b/sdk/python/tests/compiler/testdata/basic_no_decorator.py index 51c80c7d279..8e1063a5d75 100644 --- a/sdk/python/tests/compiler/testdata/basic_no_decorator.py +++ b/sdk/python/tests/compiler/testdata/basic_no_decorator.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -import kfp.gcp as gcp +import kfp.deprecated.dsl as dsl +import kfp.deprecated.gcp as gcp message_param = dsl.PipelineParam(name='message') output_path_param = dsl.PipelineParam(name='outputpath', value='default_output') diff --git a/sdk/python/tests/compiler/testdata/coin.py b/sdk/python/tests/compiler/testdata/coin.py index aecab457375..a1f6615217d 100644 --- a/sdk/python/tests/compiler/testdata/coin.py +++ b/sdk/python/tests/compiler/testdata/coin.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl class FlipCoinOp(dsl.ContainerOp): diff --git a/sdk/python/tests/compiler/testdata/compose.py b/sdk/python/tests/compiler/testdata/compose.py index 55184b5d177..44049e4b2e8 100644 --- a/sdk/python/tests/compiler/testdata/compose.py +++ b/sdk/python/tests/compiler/testdata/compose.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl class GetFrequentWordOp(dsl.ContainerOp): diff --git a/sdk/python/tests/compiler/testdata/default_value.py b/sdk/python/tests/compiler/testdata/default_value.py index bb4e69766d0..b6f08dc4709 100644 --- a/sdk/python/tests/compiler/testdata/default_value.py +++ b/sdk/python/tests/compiler/testdata/default_value.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl @dsl.pipeline( diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py index 8d4144567ab..6e50fa94d08 100644 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py +++ b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py @@ -18,8 +18,8 @@ sys.path.insert(0, __file__ + '/../../../../') -import kfp -from kfp import dsl +import kfp.deprecated as kfp +from kfp.deprecated import dsl def component_with_inline_input_artifact(text: str): diff --git a/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.py b/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.py index 739def45670..37de342c592 100644 --- a/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.py +++ b/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp import dsl -from kfp.dsl import _for_loop +import kfp.deprecated as kfp +from kfp.deprecated import dsl +from kfp.deprecated.dsl import _for_loop produce_op = kfp.components.load_component_from_text('''\ name: Produce list diff --git a/sdk/python/tests/compiler/testdata/opsgroups.py b/sdk/python/tests/compiler/testdata/opsgroups.py index b81cb6edea0..f9c47f05a76 100644 --- a/sdk/python/tests/compiler/testdata/opsgroups.py +++ b/sdk/python/tests/compiler/testdata/opsgroups.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -import kfp.dsl as dsl +import kfp.deprecated as kfp +import kfp.deprecated.dsl as dsl @dsl.graph_component diff --git a/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.py b/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.py index 0e3c329b81a..69e19898c55 100644 --- a/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.py +++ b/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.components import func_to_container_op +import kfp.deprecated as kfp +from kfp.deprecated.components import func_to_container_op @func_to_container_op @@ -70,6 +70,6 @@ def parallelfor_item_argument_resolving(): if __name__ == '__main__': - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(parallelfor_item_argument_resolving, __file__ + '.yaml') diff --git a/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.py b/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.py index 81917cbe347..675ae60172b 100644 --- a/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.py +++ b/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.components import func_to_container_op +import kfp.deprecated as kfp +from kfp.deprecated.components import func_to_container_op @func_to_container_op @@ -61,6 +61,6 @@ def parallelfor_pipeline_param_in_items_resolving(fname1: str, fname2: str): if __name__ == '__main__': - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(parallelfor_pipeline_param_in_items_resolving, __file__ + '.yaml') diff --git a/sdk/python/tests/compiler/testdata/param_op_transform.py b/sdk/python/tests/compiler/testdata/param_op_transform.py index aa2a11cb2ef..3b0d8efe40b 100644 --- a/sdk/python/tests/compiler/testdata/param_op_transform.py +++ b/sdk/python/tests/compiler/testdata/param_op_transform.py @@ -1,6 +1,6 @@ from typing import Callable -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl def add_common_labels(param): @@ -25,5 +25,5 @@ def param_substitutions(param): if __name__ == '__main__': - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(param_substitutions, __file__ + '.yaml') diff --git a/sdk/python/tests/compiler/testdata/param_substitutions.py b/sdk/python/tests/compiler/testdata/param_substitutions.py index 193b498f12f..8a82bc1ed70 100644 --- a/sdk/python/tests/compiler/testdata/param_substitutions.py +++ b/sdk/python/tests/compiler/testdata/param_substitutions.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl @dsl.pipeline( @@ -31,5 +31,5 @@ def param_substitutions(): if __name__ == '__main__': - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(param_substitutions, __file__ + '.tar.gz') diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.py b/sdk/python/tests/compiler/testdata/pipelineparams.py index 52e1c713ea5..b36ac58c19b 100644 --- a/sdk/python/tests/compiler/testdata/pipelineparams.py +++ b/sdk/python/tests/compiler/testdata/pipelineparams.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl from kubernetes import client as k8s_client from kubernetes.client.models import V1EnvVar diff --git a/sdk/python/tests/compiler/testdata/recursive_do_while.py b/sdk/python/tests/compiler/testdata/recursive_do_while.py index b107d45e9f4..9f9b6c9503d 100644 --- a/sdk/python/tests/compiler/testdata/recursive_do_while.py +++ b/sdk/python/tests/compiler/testdata/recursive_do_while.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.dsl import graph_component +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import graph_component class FlipCoinOp(dsl.ContainerOp): @@ -61,5 +61,5 @@ def recursive(): if __name__ == '__main__': - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(recursive, __file__ + '.tar.gz') diff --git a/sdk/python/tests/compiler/testdata/recursive_while.py b/sdk/python/tests/compiler/testdata/recursive_while.py index 76f78f9eff4..a4f7df99d84 100644 --- a/sdk/python/tests/compiler/testdata/recursive_while.py +++ b/sdk/python/tests/compiler/testdata/recursive_while.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl class FlipCoinOp(dsl.ContainerOp): @@ -60,5 +60,5 @@ def flipcoin(maxVal=12): if __name__ == '__main__': - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(flipcoin, __file__ + '.tar.gz') diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.py b/sdk/python/tests/compiler/testdata/resourceop_basic.py index cee86589ba3..03a3bf466b2 100644 --- a/sdk/python/tests/compiler/testdata/resourceop_basic.py +++ b/sdk/python/tests/compiler/testdata/resourceop_basic.py @@ -18,7 +18,7 @@ """ from kubernetes import client as k8s_client -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl @dsl.pipeline( @@ -51,5 +51,5 @@ def resourceop_basic(username, password): if __name__ == "__main__": - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(resourceop_basic, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/sidecar.py b/sdk/python/tests/compiler/testdata/sidecar.py index d048ff3eff3..2ec5e1fb56d 100644 --- a/sdk/python/tests/compiler/testdata/sidecar.py +++ b/sdk/python/tests/compiler/testdata/sidecar.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl from kubernetes import client as k8s_client diff --git a/sdk/python/tests/compiler/testdata/testpackage/mypipeline/compose.py b/sdk/python/tests/compiler/testdata/testpackage/mypipeline/compose.py index 55184b5d177..44049e4b2e8 100644 --- a/sdk/python/tests/compiler/testdata/testpackage/mypipeline/compose.py +++ b/sdk/python/tests/compiler/testdata/testpackage/mypipeline/compose.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl class GetFrequentWordOp(dsl.ContainerOp): diff --git a/sdk/python/tests/compiler/testdata/timeout.py b/sdk/python/tests/compiler/testdata/timeout.py index 7e00542f82a..3db8e3743ca 100755 --- a/sdk/python/tests/compiler/testdata/timeout.py +++ b/sdk/python/tests/compiler/testdata/timeout.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl class RandomFailure1Op(dsl.ContainerOp): @@ -40,5 +40,5 @@ def retry_sample_pipeline(): if __name__ == '__main__': - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(retry_sample_pipeline, __file__ + '.tar.gz') diff --git a/sdk/python/tests/compiler/testdata/two_step.py b/sdk/python/tests/compiler/testdata/two_step.py index 1a550202862..a641ee4a967 100644 --- a/sdk/python/tests/compiler/testdata/two_step.py +++ b/sdk/python/tests/compiler/testdata/two_step.py @@ -13,8 +13,8 @@ # limitations under the License. """Two step v2-compatible pipeline.""" -from kfp import components, dsl -from kfp.components import InputPath, OutputPath +from kfp.deprecated import components, dsl +from kfp.deprecated.components import InputPath, OutputPath def preprocess(uri: str, some_int: int, output_parameter_one: OutputPath(int), diff --git a/sdk/python/tests/compiler/testdata/uri_artifacts.py b/sdk/python/tests/compiler/testdata/uri_artifacts.py index f33f5efb227..f969c59385b 100644 --- a/sdk/python/tests/compiler/testdata/uri_artifacts.py +++ b/sdk/python/tests/compiler/testdata/uri_artifacts.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. """Pipeline DSL code for testing URI-based artifact passing.""" -from kfp import compiler -from kfp import components -from kfp import dsl +from kfp.deprecated import compiler +from kfp.deprecated import components +from kfp.deprecated import dsl # Patch to make the test result deterministic. diff --git a/sdk/python/tests/compiler/testdata/volume.py b/sdk/python/tests/compiler/testdata/volume.py index fbd323bbfd1..948a45004c9 100644 --- a/sdk/python/tests/compiler/testdata/volume.py +++ b/sdk/python/tests/compiler/testdata/volume.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl from kubernetes import client as k8s_client diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py index 41ce174889f..4ba7d60ebc3 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py @@ -19,7 +19,7 @@ such way is widespread in storage systems integrated with K8s. """ -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl @dsl.pipeline( @@ -72,5 +72,5 @@ def volume_snapshotop_rokurl(rok_url): if __name__ == "__main__": - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(volume_snapshotop_rokurl, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py index d7541f5f308..94ea448cb6d 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl @dsl.pipeline( @@ -72,6 +72,6 @@ def volume_snapshotop_sequential(url): if __name__ == "__main__": - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(volume_snapshotop_sequential, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_basic.py b/sdk/python/tests/compiler/testdata/volumeop_basic.py index 90a6609b232..fe9d5465e54 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_basic.py +++ b/sdk/python/tests/compiler/testdata/volumeop_basic.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl @dsl.pipeline( @@ -33,5 +33,5 @@ def volumeop_basic(size): if __name__ == "__main__": - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(volumeop_basic, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_dag.py b/sdk/python/tests/compiler/testdata/volumeop_dag.py index 83dd33dde21..4e37016aff7 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_dag.py +++ b/sdk/python/tests/compiler/testdata/volumeop_dag.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl @dsl.pipeline( @@ -47,5 +47,5 @@ def volume_op_dag(): if __name__ == "__main__": - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(volume_op_dag, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_parallel.py b/sdk/python/tests/compiler/testdata/volumeop_parallel.py index a22c3d03538..53b5c63309c 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_parallel.py +++ b/sdk/python/tests/compiler/testdata/volumeop_parallel.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl @dsl.pipeline( @@ -48,5 +48,5 @@ def volumeop_parallel(): if __name__ == "__main__": - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(volumeop_parallel, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_sequential.py b/sdk/python/tests/compiler/testdata/volumeop_sequential.py index 986c850ff6b..1945f09bc3a 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_sequential.py +++ b/sdk/python/tests/compiler/testdata/volumeop_sequential.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl @dsl.pipeline( @@ -47,5 +47,5 @@ def volumeop_sequential(): if __name__ == "__main__": - import kfp.compiler as compiler + import kfp.deprecated.compiler as compiler compiler.Compiler().compile(volumeop_sequential, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/withitem_basic.py b/sdk/python/tests/compiler/testdata/withitem_basic.py index ac2bc34bdf7..542a8e26eff 100644 --- a/sdk/python/tests/compiler/testdata/withitem_basic.py +++ b/sdk/python/tests/compiler/testdata/withitem_basic.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.dsl import _for_loop +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import _for_loop @dsl.pipeline(name='my-pipeline') @@ -43,10 +43,10 @@ def pipeline(my_pipe_param: int = 10): if __name__ == '__main__': - from kfp import compiler + from kfp.deprecated import compiler print(compiler.Compiler().compile(pipeline, package_path=None)) - import kfp + import kfp.deprecated as kfp client = kfp.Client(host='127.0.0.1:8080/pipeline') pkg_path = '/tmp/witest_pkg.tar.gz' diff --git a/sdk/python/tests/compiler/testdata/withitem_nested.py b/sdk/python/tests/compiler/testdata/withitem_nested.py index dd582a31ca5..922370cea81 100644 --- a/sdk/python/tests/compiler/testdata/withitem_nested.py +++ b/sdk/python/tests/compiler/testdata/withitem_nested.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.dsl import _for_loop +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import _for_loop @dsl.pipeline(name='my-pipeline') @@ -53,8 +53,8 @@ def pipeline(my_pipe_param: int = 10): if __name__ == '__main__': - from kfp import compiler - import kfp + from kfp.deprecated import compiler + import kfp.deprecated as kfp import time client = kfp.Client(host='127.0.0.1:8080/pipeline') print(compiler.Compiler().compile(pipeline, package_path=None)) diff --git a/sdk/python/tests/compiler/testdata/withparam_global.py b/sdk/python/tests/compiler/testdata/withparam_global.py index 0674e47608c..dbf35216e55 100644 --- a/sdk/python/tests/compiler/testdata/withparam_global.py +++ b/sdk/python/tests/compiler/testdata/withparam_global.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.dsl import _for_loop +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import _for_loop @dsl.pipeline(name='my-pipeline') @@ -45,8 +45,8 @@ def pipeline(loopidy_doop: list = [3, 5, 7, 9]): if __name__ == '__main__': - from kfp import compiler - import kfp + from kfp.deprecated import compiler + import kfp.deprecated as kfp import time client = kfp.Client(host='127.0.0.1:8080/pipeline') print(compiler.Compiler().compile(pipeline, package_path=None)) diff --git a/sdk/python/tests/compiler/testdata/withparam_global_dict.py b/sdk/python/tests/compiler/testdata/withparam_global_dict.py index 8377ad464d4..5b75e642dc9 100644 --- a/sdk/python/tests/compiler/testdata/withparam_global_dict.py +++ b/sdk/python/tests/compiler/testdata/withparam_global_dict.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.dsl import _for_loop +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import _for_loop @dsl.pipeline(name='my-pipeline') @@ -45,8 +45,8 @@ def pipeline(loopidy_doop: dict = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]): if __name__ == '__main__': - from kfp import compiler - import kfp + from kfp.deprecated import compiler + import kfp.deprecated as kfp import time client = kfp.Client(host='127.0.0.1:8080/pipeline') print(compiler.Compiler().compile(pipeline, package_path=None)) diff --git a/sdk/python/tests/compiler/testdata/withparam_output.py b/sdk/python/tests/compiler/testdata/withparam_output.py index c5c10386711..cd0abbd2f81 100644 --- a/sdk/python/tests/compiler/testdata/withparam_output.py +++ b/sdk/python/tests/compiler/testdata/withparam_output.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.dsl import _for_loop +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import _for_loop @dsl.pipeline(name='my-pipeline') @@ -45,8 +45,8 @@ def pipeline(): if __name__ == '__main__': - from kfp import compiler - import kfp + from kfp.deprecated import compiler + import kfp.deprecated as kfp import time client = kfp.Client(host='127.0.0.1:8080/pipeline') print(compiler.Compiler().compile(pipeline, package_path=None)) diff --git a/sdk/python/tests/compiler/testdata/withparam_output_dict.py b/sdk/python/tests/compiler/testdata/withparam_output_dict.py index 63a75118aa7..ac37778bc90 100644 --- a/sdk/python/tests/compiler/testdata/withparam_output_dict.py +++ b/sdk/python/tests/compiler/testdata/withparam_output_dict.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.dsl import _for_loop +import kfp.deprecated.dsl as dsl +from kfp.deprecated.dsl import _for_loop @dsl.pipeline(name='my-pipeline') @@ -45,8 +45,8 @@ def pipeline(): if __name__ == '__main__': - from kfp import compiler - import kfp + from kfp.deprecated import compiler + import kfp.deprecated as kfp import time client = kfp.Client(host='127.0.0.1:8080/pipeline') print(compiler.Compiler().compile(pipeline, package_path=None)) diff --git a/sdk/python/tests/dsl/component_bridge_tests.py b/sdk/python/tests/dsl/component_bridge_tests.py index 0426a87b8a9..0554478d877 100644 --- a/sdk/python/tests/dsl/component_bridge_tests.py +++ b/sdk/python/tests/dsl/component_bridge_tests.py @@ -160,7 +160,7 @@ def test_type_compatibility_check_not_failing_when_type_is_ignored(self): b_task = task_factory_b(in1=a_task.outputs['out1'].ignore_type()) def test_end_to_end_python_component_pipeline_compilation(self): - import kfp.components as comp + import kfp.deprecated.components as comp #Defining the Python function def add(a: float, b: float) -> float: @@ -207,7 +207,7 @@ def test_handling_list_arguments_containing_pipelineparam(self): def consume_list(list_param: list) -> int: pass - import kfp + import kfp.deprecated as kfp task_factory = create_component_from_func(consume_list) task = task_factory([1, 2, 3, kfp.dsl.PipelineParam('aaa'), 4, 5, 6]) diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index cb3fb865f5e..8b135ce8640 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -118,8 +118,8 @@ def componentA(a: float, b: List[int], c: Optional[str] = None) -> None: def test_python_component_decorator(self): # Deprecated - from kfp.dsl import python_component - from kfp.components import create_component_from_func + from kfp.deprecated.dsl import python_component + from kfp.deprecated.components import create_component_from_func expected_name = 'Sum component name' expected_description = 'Sum component description' diff --git a/sdk/python/tests/dsl/extensions/test_kubernetes.py b/sdk/python/tests/dsl/extensions/test_kubernetes.py index f26c500ccf1..dfd6fc52f5a 100644 --- a/sdk/python/tests/dsl/extensions/test_kubernetes.py +++ b/sdk/python/tests/dsl/extensions/test_kubernetes.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.dsl import ContainerOp -from kfp.dsl.extensions.kubernetes import use_secret +from kfp.deprecated.dsl import ContainerOp +from kfp.deprecated.dsl.extensions.kubernetes import use_secret import unittest import inspect diff --git a/test/kfp-functional-test/run_kfp_functional_test.py b/test/kfp-functional-test/run_kfp_functional_test.py index 369fd62ef15..faf1083a9c9 100644 --- a/test/kfp-functional-test/run_kfp_functional_test.py +++ b/test/kfp-functional-test/run_kfp_functional_test.py @@ -17,8 +17,8 @@ import string from datetime import datetime -import kfp -from kfp import dsl +import kfp.deprecated as kfp +from kfp.deprecated import dsl import constants diff --git a/test/sample-test/check_notebook_results.py b/test/sample-test/check_notebook_results.py index 15d0fcb5fe1..1aa2a0ad2fb 100644 --- a/test/sample-test/check_notebook_results.py +++ b/test/sample-test/check_notebook_results.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp import subprocess import utils import yaml from constants import RUN_LIST_PAGE_SIZE, DEFAULT_CONFIG -from kfp import Client +from kfp.deprecated import Client class NoteBookChecker(object): diff --git a/test/sample-test/run_resnet_cmle_test.py b/test/sample-test/run_resnet_cmle_test.py index bce46c13409..249d0b8ce7e 100644 --- a/test/sample-test/run_resnet_cmle_test.py +++ b/test/sample-test/run_resnet_cmle_test.py @@ -18,7 +18,7 @@ import tarfile from datetime import datetime import utils -from kfp import Client +from kfp.deprecated import Client ###### Input/Output Instruction ###### # input: yaml diff --git a/test/sample-test/run_sample_test.py b/test/sample-test/run_sample_test.py index 2ec17652171..dd9300c1c2e 100644 --- a/test/sample-test/run_sample_test.py +++ b/test/sample-test/run_sample_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp import os import tarfile import time @@ -20,7 +20,7 @@ import yamale import yaml from datetime import datetime -from kfp import Client +from kfp.deprecated import Client from constants import CONFIG_DIR, DEFAULT_CONFIG, SCHEMA_CONFIG diff --git a/test/sample-test/sample_test_launcher.py b/test/sample-test/sample_test_launcher.py index f9bd30a27ff..cc4ea069ca1 100644 --- a/test/sample-test/sample_test_launcher.py +++ b/test/sample-test/sample_test_launcher.py @@ -28,7 +28,7 @@ from constants import PAPERMILL_ERR_MSG, BASE_DIR, TEST_DIR, SCHEMA_CONFIG, CONFIG_DIR, DEFAULT_CONFIG from check_notebook_results import NoteBookChecker -from kfp.containers._gcs_helper import GCSHelper +from kfp.deprecated.containers._gcs_helper import GCSHelper from run_sample_test import PySampleChecker From d39bb62bc6afdbd46bc8c3da7109aaf5bc11d22e Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 9 Feb 2022 19:48:54 -0800 Subject: [PATCH 06/39] s' --- sdk/python/kfp/deprecated/compiler/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/kfp/deprecated/compiler/main.py b/sdk/python/kfp/deprecated/compiler/main.py index 03bafa5dc95..7b6434545fd 100644 --- a/sdk/python/kfp/deprecated/compiler/main.py +++ b/sdk/python/kfp/deprecated/compiler/main.py @@ -15,6 +15,7 @@ import argparse from typing import Optional from kfp.deprecated import dsl +import kfp.deprecated as kfp import kfp.deprecated.compiler import os import sys From 9960a2f668feecde8d79a325ec86912213555b35 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 9 Feb 2022 20:09:44 -0800 Subject: [PATCH 07/39] fix tests --- .../kfp/deprecated/compiler/v2_compat.py | 1 - ...htweight_python_functions_v2_pipeline.json | 6 ++-- ...ight_python_functions_v2_with_outputs.json | 10 +++---- .../test_data/pipeline_with_after.json | 2 +- .../pipeline_with_concat_placeholder.json | 2 +- .../test_data/pipeline_with_condition.json | 12 ++++---- .../test_data/pipeline_with_env.json | 4 +-- .../test_data/pipeline_with_exit_handler.json | 8 +++--- .../test_data/pipeline_with_gcpc_types.json | 4 +-- .../pipeline_with_if_placeholder.json | 2 +- .../test_data/pipeline_with_importer.json | 4 +-- .../test_data/pipeline_with_loops.json | 18 ++++++------ .../pipeline_with_loops_and_conditions.json | 28 +++++++++---------- .../pipeline_with_metrics_outputs.json | 6 ++-- .../pipeline_with_nested_conditions.json | 18 ++++++------ .../pipeline_with_nested_conditions_yaml.json | 2 +- .../test_data/pipeline_with_nested_loops.json | 8 +++--- .../test_data/pipeline_with_ontology.json | 2 +- ...ipeline_with_params_containing_format.json | 8 +++--- .../test_data/pipeline_with_placeholders.json | 12 ++++---- .../pipeline_with_resource_spec.json | 2 +- .../pipeline_with_reused_component.json | 2 +- .../pipeline_with_various_io_types.json | 2 +- .../test_data/two_step_pipeline.json | 2 +- .../v2_component_with_optional_inputs.json | 4 +-- .../test_data/xgboost_sample_pipeline.json | 2 +- 26 files changed, 85 insertions(+), 86 deletions(-) diff --git a/sdk/python/kfp/deprecated/compiler/v2_compat.py b/sdk/python/kfp/deprecated/compiler/v2_compat.py index 00c513730f2..03af5d7fae9 100644 --- a/sdk/python/kfp/deprecated/compiler/v2_compat.py +++ b/sdk/python/kfp/deprecated/compiler/v2_compat.py @@ -19,7 +19,6 @@ from kfp.deprecated import dsl from kfp.deprecated.compiler import _default_transformers from kfp.pipeline_spec import pipeline_spec_pb2 -from kfp.v2 import compiler from kubernetes import client as k8s_client _DEFAULT_LAUNCHER_IMAGE = "gcr.io/ml-pipeline/kfp-launcher:1.8.7" diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json index 7855e934a16..3a4f1b620be 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json @@ -104,7 +104,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -124,7 +124,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -245,5 +245,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json index 68bf591c379..a62d96495c7 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json @@ -112,7 +112,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -132,7 +132,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -152,7 +152,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -172,7 +172,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -328,5 +328,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json index e70c21d9f07..7b528973fb6 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json @@ -146,5 +146,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-2.0.0-beta0" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json index 780ba6bc193..a8588ccfcef 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json @@ -55,5 +55,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json index 47a74ef6d69..cad42d7bf72 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json @@ -132,7 +132,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -152,7 +152,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -172,7 +172,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -192,7 +192,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -212,7 +212,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -301,5 +301,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json index e2497600d73..29230e105a9 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json @@ -41,7 +41,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -90,5 +90,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json index 8baae70ed49..0287adb21b1 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json @@ -95,7 +95,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -115,7 +115,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -135,7 +135,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -205,5 +205,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json index 1582238f6ac..e223f96e826 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json @@ -40,7 +40,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -107,5 +107,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json index 32665fd1e22..222eb9c1874 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json @@ -76,5 +76,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json index bc153983589..8cd7a589bee 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json @@ -195,7 +195,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.11' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -215,7 +215,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.11' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json index 122b9d9d50c..1d157072c4c 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json @@ -269,7 +269,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -289,7 +289,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -309,7 +309,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -329,7 +329,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -349,7 +349,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -369,7 +369,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -389,7 +389,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -409,7 +409,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -510,5 +510,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json index 06e619ab2dd..cb478caca27 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json @@ -923,7 +923,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -943,7 +943,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -963,7 +963,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -983,7 +983,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1003,7 +1003,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1023,7 +1023,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1043,7 +1043,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1063,7 +1063,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1083,7 +1083,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1103,7 +1103,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1123,7 +1123,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1143,7 +1143,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1163,7 +1163,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -1292,5 +1292,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json index 9814880b180..d7e8cac3e5d 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json @@ -87,7 +87,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -107,7 +107,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -189,5 +189,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json index 99e20c5653f..8cd4d42c0b5 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json @@ -229,7 +229,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -249,7 +249,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -269,7 +269,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -289,7 +289,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -309,7 +309,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -329,7 +329,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -349,7 +349,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -369,7 +369,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -491,5 +491,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json index c960bb3a59e..78fc3594abf 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json @@ -536,5 +536,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-2.0.0-beta0" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json index b7528b0dfb3..5f0ff0ad4db 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json @@ -215,7 +215,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -235,7 +235,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -255,7 +255,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -342,5 +342,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json index c97f55c7b60..4efd952a14d 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json @@ -159,5 +159,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json index c61c732bcb6..3a260927888 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json @@ -110,7 +110,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -130,7 +130,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -150,7 +150,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -251,5 +251,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_placeholders.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_placeholders.json index 6ceae17a555..2e764731551 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_placeholders.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_placeholders.json @@ -79,7 +79,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -99,7 +99,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -119,7 +119,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -139,7 +139,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -159,7 +159,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -305,5 +305,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json index a6aacf502ee..80182293c30 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json @@ -167,5 +167,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json index 1fe025cd751..ce95f2d0c16 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json @@ -206,5 +206,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json index 5b4eeda5f05..855d23b03ed 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json @@ -303,5 +303,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json index 63628409903..6b78e984a0d 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json @@ -122,5 +122,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json index d3bac542b23..fcef5d3b738 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json @@ -27,7 +27,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.9' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -73,5 +73,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json index df118cf6a5e..57129abf9b0 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json @@ -755,5 +755,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.9" + "sdkVersion": "kfp-1.8.11" } \ No newline at end of file From 3376f46c122b3b452916ffc9ab758e32368ec9e6 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Sat, 12 Feb 2022 23:54:08 -0800 Subject: [PATCH 08/39] fix test --- .../kfp/deprecated/cli/components_test.py | 4 +- .../kfp/deprecated/compiler/__init__.py | 4 +- sdk/python/kfp/deprecated/compiler/main.py | 2 +- .../compiler/v2_compatible_compiler_test.py | 2 +- sdk/python/kfp/deprecated/dsl/_component.py | 2 +- .../kfp/deprecated/dsl/_container_op.py | 1 + .../kfp/deprecated/dsl/_container_op_test.py | 2 +- sdk/python/kfp/deprecated/notebook/_magic.py | 2 +- .../kfp/v2/components/types/type_utils.py | 2 +- .../v2/components/types/type_utils_test.py | 2 +- sdk/python/tests/compiler/compiler_tests.py | 1515 ----------------- .../tests/compiler/container_builder_test.py | 2 +- 12 files changed, 13 insertions(+), 1527 deletions(-) delete mode 100644 sdk/python/tests/compiler/compiler_tests.py diff --git a/sdk/python/kfp/deprecated/cli/components_test.py b/sdk/python/kfp/deprecated/cli/components_test.py index 5b1881f3a5b..5bbd7867f91 100644 --- a/sdk/python/kfp/deprecated/cli/components_test.py +++ b/sdk/python/kfp/deprecated/cli/components_test.py @@ -409,7 +409,7 @@ def testDockerfileIsCreatedCorrectly(self): COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt - RUN pip install --no-cache-dir kfp==1.2.3 + RUN pip install --no-cache-dir kfp==1.8.11 COPY . . ''')) @@ -452,7 +452,7 @@ def testExistingDockerfileCanBeOverwritten(self): COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt - RUN pip install --no-cache-dir kfp==1.2.3 + RUN pip install --no-cache-dir kfp==1.8.11 COPY . . ''')) diff --git a/sdk/python/kfp/deprecated/compiler/__init__.py b/sdk/python/kfp/deprecated/compiler/__init__.py index 519a1c8711a..248088cb6fd 100644 --- a/sdk/python/kfp/deprecated/compiler/__init__.py +++ b/sdk/python/kfp/deprecated/compiler/__init__.py @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .compiler import Compiler -from ..containers._component_builder import build_python_component, build_docker_image, VersionedDependency +from kfp.deprecated.compiler.compiler import Compiler +from kfp.deprecated.containers._component_builder import build_python_component, build_docker_image, VersionedDependency diff --git a/sdk/python/kfp/deprecated/compiler/main.py b/sdk/python/kfp/deprecated/compiler/main.py index 7b6434545fd..ef15c21221c 100644 --- a/sdk/python/kfp/deprecated/compiler/main.py +++ b/sdk/python/kfp/deprecated/compiler/main.py @@ -83,7 +83,7 @@ def _compile_pipeline_function( else: pipeline_func = pipeline_funcs[0] - kfp.compiler.Compiler(mode=mode).compile(pipeline_func, output_path, + kfp.deprecated.compiler.Compiler(mode=mode).compile(pipeline_func, output_path, type_check, pipeline_conf) diff --git a/sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py b/sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py index e41be0566bf..975a9936864 100644 --- a/sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py +++ b/sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py @@ -21,7 +21,7 @@ from typing import Callable, Dict import yaml -from kfp.deprecated import compiler, components +from kfp.deprecated import compiler from kfp.deprecated import dsl as v1dsl from kfp.v2 import dsl from kfp.v2.dsl import Artifact, InputPath, OutputPath, component diff --git a/sdk/python/kfp/deprecated/dsl/_component.py b/sdk/python/kfp/deprecated/dsl/_component.py index 5a08038072b..b490ad51f06 100644 --- a/sdk/python/kfp/deprecated/dsl/_component.py +++ b/sdk/python/kfp/deprecated/dsl/_component.py @@ -17,7 +17,7 @@ from ._pipeline_param import PipelineParam from .types import check_types, InconsistentTypeException from ._ops_group import Graph -import kfp.deprecated +import kfp.deprecated as kfp @deprecated( diff --git a/sdk/python/kfp/deprecated/dsl/_container_op.py b/sdk/python/kfp/deprecated/dsl/_container_op.py index 83f28cf7483..b156dd5f4b2 100644 --- a/sdk/python/kfp/deprecated/dsl/_container_op.py +++ b/sdk/python/kfp/deprecated/dsl/_container_op.py @@ -28,6 +28,7 @@ V1SecurityContext, V1Volume, V1VolumeDevice, V1VolumeMount) + # generics T = TypeVar('T') # type alias: either a string or a list of string diff --git a/sdk/python/kfp/deprecated/dsl/_container_op_test.py b/sdk/python/kfp/deprecated/dsl/_container_op_test.py index 43d0bd2fd1c..a1f4f743f02 100644 --- a/sdk/python/kfp/deprecated/dsl/_container_op_test.py +++ b/sdk/python/kfp/deprecated/dsl/_container_op_test.py @@ -15,7 +15,7 @@ import unittest from kfp.deprecated.dsl import _container_op -from kfp.deprecated.pipeline_spec import pipeline_spec_pb2 +from kfp.pipeline_spec import pipeline_spec_pb2 from google.protobuf import text_format from google.protobuf import json_format diff --git a/sdk/python/kfp/deprecated/notebook/_magic.py b/sdk/python/kfp/deprecated/notebook/_magic.py index 203857580e9..dedfdc910af 100644 --- a/sdk/python/kfp/deprecated/notebook/_magic.py +++ b/sdk/python/kfp/deprecated/notebook/_magic.py @@ -17,7 +17,7 @@ from deprecated.sphinx import deprecated -from ..compiler import build_docker_image +from kfp.deprecated.compiler import build_docker_image @deprecated( diff --git a/sdk/python/kfp/v2/components/types/type_utils.py b/sdk/python/kfp/v2/components/types/type_utils.py index d120b54ba9e..95bf1525a19 100644 --- a/sdk/python/kfp/v2/components/types/type_utils.py +++ b/sdk/python/kfp/v2/components/types/type_utils.py @@ -234,7 +234,7 @@ def verify_type_compatibility( error_text = error_message_prefix + ( 'Argument type "{}" is incompatible with the input type "{}"' ).format(str(given_type), str(expected_type)) - import kfp + import kfp.deprecated as kfp if kfp.TYPE_CHECK: raise InconsistentTypeException(error_text) else: diff --git a/sdk/python/kfp/v2/components/types/type_utils_test.py b/sdk/python/kfp/v2/components/types/type_utils_test.py index 8cc06914805..7a622c48da5 100644 --- a/sdk/python/kfp/v2/components/types/type_utils_test.py +++ b/sdk/python/kfp/v2/components/types/type_utils_test.py @@ -17,7 +17,7 @@ from typing import Any, Dict, List, Union from absl.testing import parameterized -from kfp.components import structures +from kfp.deprecated.components import structures from kfp.pipeline_spec import pipeline_spec_pb2 as pb from kfp.v2.components.types import artifact_types, type_utils from kfp.v2.components.types.type_utils import InconsistentTypeException diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py deleted file mode 100644 index e2909164c35..00000000000 --- a/sdk/python/tests/compiler/compiler_tests.py +++ /dev/null @@ -1,1515 +0,0 @@ -# Copyright 2018-2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Optional -import unittest - -import kfp.deprecated as kfp -import kfp.deprecated.compiler as compiler -import kfp.deprecated.dsl as dsl -import json -import os -import shutil -import subprocess -import sys -import zipfile -import tarfile -import tempfile -import yaml -from unittest import mock - -from absl.testing import parameterized -from kfp.deprecated.compiler import Compiler -from kfp.deprecated.dsl._component import component -from kfp.deprecated.dsl import ContainerOp, pipeline, PipelineParam -from kfp.deprecated.dsl.types import Integer, InconsistentTypeException -from kubernetes.client import V1Toleration, V1Affinity, V1NodeSelector, V1NodeSelectorRequirement, V1NodeSelectorTerm, \ - V1NodeAffinity, V1PodDNSConfig, V1PodDNSConfigOption - - -def some_op(): - return dsl.ContainerOp( - name='sleep', - image='busybox', - command=['sleep 1'], - ) - - -class TestCompiler(parameterized.TestCase): - # Define the places of samples covered by unit tests. - core_sample_path = os.path.join( - os.path.dirname(__file__), - '..', - '..', - '..', - '..', - 'samples', - 'core', - ) - - def test_operator_to_template(self): - """Test converting operator to template.""" - - from kubernetes import client as k8s_client - - def my_pipeline(msg1, json, kind, msg2='value2'): - op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'], - arguments=['echo %s %s | tee /tmp/message.txt' % (msg1, msg2)], - file_outputs={'merged': '/tmp/message.txt'}) \ - .add_volume_mount(k8s_client.V1VolumeMount( - mount_path='/secret/gcp-credentials', - name='gcp-credentials')) \ - .add_env_variable(k8s_client.V1EnvVar( - name='GOOGLE_APPLICATION_CREDENTIALS', - value='/secret/gcp-credentials/user-gcp-sa.json')) - res = dsl.ResourceOp( - name="test-resource", - k8s_resource=k8s_client.V1PersistentVolumeClaim( - api_version="v1", - kind=kind, - metadata=k8s_client.V1ObjectMeta(name="resource")), - attribute_outputs={"out": json}, - set_owner_reference=True) - golden_output = { - 'container': { - 'image': - 'image', - 'args': [ - 'echo {{inputs.parameters.msg1}} {{inputs.parameters.msg2}} | tee /tmp/message.txt' - ], - 'command': ['sh', '-c'], - 'env': [{ - 'name': 'GOOGLE_APPLICATION_CREDENTIALS', - 'value': '/secret/gcp-credentials/user-gcp-sa.json' - }], - 'volumeMounts': [{ - 'mountPath': '/secret/gcp-credentials', - 'name': 'gcp-credentials', - }] - }, - 'inputs': { - 'parameters': [ - { - 'name': 'msg1' - }, - { - 'name': 'msg2' - }, - ] - }, - 'name': 'echo', - 'outputs': { - 'artifacts': [{ - 'name': 'echo-merged', - 'path': '/tmp/message.txt', - },], - 'parameters': [{ - 'name': 'echo-merged', - 'valueFrom': { - 'path': '/tmp/message.txt' - } - }], - }, - 'metadata': { - 'labels': { - 'pipelines.kubeflow.org/enable_caching': 'true' - } - } - } - res_output = { - 'inputs': { - 'parameters': [{ - 'name': 'json' - }, { - 'name': 'kind' - }] - }, - 'name': 'test-resource', - 'outputs': { - 'parameters': [{ - 'name': 'test-resource-manifest', - 'valueFrom': { - 'jsonPath': '{}' - } - }, { - 'name': 'test-resource-name', - 'valueFrom': { - 'jsonPath': '{.metadata.name}' - } - }, { - 'name': 'test-resource-out', - 'valueFrom': { - 'jsonPath': '{{inputs.parameters.json}}' - } - }] - }, - 'resource': { - 'action': 'create', - 'manifest': ("apiVersion: v1\n" - "kind: '{{inputs.parameters.kind}}'\n" - "metadata:\n" - " name: resource\n"), - 'setOwnerReference': True - }, - 'metadata': { - 'labels': { - 'pipelines.kubeflow.org/enable_caching': 'true' - } - } - } - - self.maxDiff = None - self.assertEqual(golden_output, - compiler._op_to_template._op_to_template(op)) - self.assertEqual(res_output, - compiler._op_to_template._op_to_template(res)) - - kfp.compiler.Compiler()._compile(my_pipeline) - - def _get_yaml_from_zip(self, zip_file): - with zipfile.ZipFile(zip_file, 'r') as zip: - with open(zip.extract(zip.namelist()[0]), 'r') as yaml_file: - return yaml.safe_load(yaml_file) - - def _get_yaml_from_tar(self, tar_file): - with tarfile.open(tar_file, 'r:gz') as tar: - return yaml.safe_load(tar.extractfile(tar.getmembers()[0])) - - def test_basic_workflow(self): - """Test compiling a basic workflow.""" - - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - sys.path.append(test_data_dir) - import basic - tmpdir = tempfile.mkdtemp() - package_path = os.path.join(tmpdir, 'workflow.zip') - try: - compiler.Compiler().compile(basic.save_most_frequent_word, - package_path) - with open(os.path.join(test_data_dir, 'basic.yaml'), 'r') as f: - golden = yaml.safe_load(f) - compiled = self._get_yaml_from_zip(package_path) - - for workflow in golden, compiled: - del workflow['metadata'] - for template in workflow['spec']['templates']: - template.pop('metadata', None) - - self.maxDiff = None - # Comment next line for generating golden yaml. - self.assertEqual(golden, compiled) - finally: - # Replace next line with commented line for gathering golden yaml. - shutil.rmtree(tmpdir) - # print(tmpdir) - - def test_basic_workflow_without_decorator(self): - """Test compiling a workflow and appending pipeline params.""" - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - sys.path.append(test_data_dir) - import basic_no_decorator - tmpdir = tempfile.mkdtemp() - try: - compiled_workflow = compiler.Compiler().create_workflow( - basic_no_decorator.save_most_frequent_word, - 'Save Most Frequent', 'Get Most Frequent Word and Save to GCS', - [ - basic_no_decorator.message_param, - basic_no_decorator.output_path_param - ]) - with open( - os.path.join(test_data_dir, 'basic_no_decorator.yaml'), - 'r') as f: - golden = yaml.safe_load(f) - - name_to_template = { - template['name']: template - for template in compiled_workflow['spec']['templates'] - } - for k, v in name_to_template.items(): - if k in ['exiting', 'get-frequent', 'save']: - self.assertEqual( - v['metadata']['labels'] - ['pipelines.kubeflow.org/pipeline-sdk-type'], 'kfp') - self.assertTrue( - v['metadata']['labels'] - ['pipelines.kubeflow.org/kfp_sdk_version'] is not None) - - for workflow in golden, compiled_workflow: - del workflow['metadata'] - for template in workflow['spec']['templates']: - template.pop('metadata', None) - - self.assertEqual(golden, compiled_workflow) - finally: - shutil.rmtree(tmpdir) - - def test_composing_workflow(self): - """Test compiling a simple workflow, and a bigger one composed from the - simple one.""" - - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - sys.path.append(test_data_dir) - import compose - tmpdir = tempfile.mkdtemp() - try: - # First make sure the simple pipeline can be compiled. - simple_package_path = os.path.join(tmpdir, 'simple.zip') - compiler.Compiler().compile(compose.save_most_frequent_word, - simple_package_path) - - # Then make sure the composed pipeline can be compiled and also compare with golden. - compose_package_path = os.path.join(tmpdir, 'compose.zip') - compiler.Compiler().compile( - compose.download_save_most_frequent_word, compose_package_path) - with open(os.path.join(test_data_dir, 'compose.yaml'), 'r') as f: - golden = yaml.safe_load(f) - compiled = self._get_yaml_from_zip(compose_package_path) - - for workflow in golden, compiled: - del workflow['metadata'] - for template in workflow['spec']['templates']: - template.pop('metadata', None) - - self.maxDiff = None - # Comment next line for generating golden yaml. - self.assertEqual(golden, compiled) - finally: - # Replace next line with commented line for gathering golden yaml. - shutil.rmtree(tmpdir) - # print(tmpdir) - - def _test_py_compile_zip(self, file_base_name): - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - py_file = os.path.join(test_data_dir, file_base_name + '.py') - tmpdir = tempfile.mkdtemp() - try: - target_zip = os.path.join(tmpdir, file_base_name + '.zip') - subprocess.check_call( - ['dsl-compile', '--py', py_file, '--output', target_zip]) - with open( - os.path.join(test_data_dir, file_base_name + '.yaml'), - 'r') as f: - golden = yaml.safe_load(f) - compiled = self._get_yaml_from_zip(target_zip) - - for workflow in golden, compiled: - del workflow['metadata'] - for template in workflow['spec']['templates']: - template.pop('metadata', None) - - self.maxDiff = None - self.assertEqual(golden, compiled) - finally: - shutil.rmtree(tmpdir) - - def _test_py_compile_targz(self, file_base_name): - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - py_file = os.path.join(test_data_dir, file_base_name + '.py') - tmpdir = tempfile.mkdtemp() - try: - target_tar = os.path.join(tmpdir, file_base_name + '.tar.gz') - subprocess.check_call( - ['dsl-compile', '--py', py_file, '--output', target_tar]) - with open( - os.path.join(test_data_dir, file_base_name + '.yaml'), - 'r') as f: - golden = yaml.safe_load(f) - compiled = self._get_yaml_from_tar(target_tar) - - for workflow in golden, compiled: - del workflow['metadata'] - for template in workflow['spec']['templates']: - template.pop('metadata', None) - - self.maxDiff = None - self.assertEqual(golden, compiled) - finally: - shutil.rmtree(tmpdir) - - def _test_py_compile_yaml(self, - file_base_name: str, - mode: Optional[str] = None): - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - py_file = os.path.join(test_data_dir, file_base_name + '.py') - tmpdir = tempfile.mkdtemp() - try: - target_yaml = os.path.join(tmpdir, - file_base_name + '-pipeline.yaml') - cmds = ['dsl-compile', '--py', py_file, '--output', target_yaml] - if mode: - cmds.extend(['--mode', mode]) - subprocess.check_call(cmds) - with open( - os.path.join(test_data_dir, file_base_name + '.yaml'), - 'r') as f: - golden = yaml.safe_load(f) - - with open(os.path.join(test_data_dir, target_yaml), 'r') as f: - compiled = yaml.safe_load(f) - - for workflow in golden, compiled: - del workflow['metadata'] - for template in workflow['spec']['templates']: - template.pop('metadata', None) - - # v2-compat mode uses launcher image with pinned version. Ignore it. - if 'initContainers' in template and ( - template['initContainers'][0]['image'].startswith( - 'gcr.io/ml-pipeline/kfp-launcher')): - template['initContainers'][0].pop('image', None) - - self.maxDiff = None - self.assertEqual(golden, compiled) - finally: - shutil.rmtree(tmpdir) - - def _test_sample_py_compile_yaml(self, file_base_name): - # Jump back to sample dir for sample python file. - sample_data_dir = os.path.join(self.core_sample_path, file_base_name) - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - py_file = os.path.join(sample_data_dir, file_base_name + '.py') - tmpdir = tempfile.mkdtemp() - try: - target_yaml = os.path.join(tmpdir, - file_base_name + '-pipeline.yaml') - subprocess.check_call( - ['dsl-compile', '--py', py_file, '--output', target_yaml]) - with open( - os.path.join(test_data_dir, file_base_name + '.yaml'), - 'r') as f: - golden = yaml.safe_load(f) - - with open(os.path.join(test_data_dir, target_yaml), 'r') as f: - compiled = yaml.safe_load(f) - - for workflow in golden, compiled: - del workflow['metadata'] - for template in workflow['spec']['templates']: - template.pop('metadata', None) - - self.maxDiff = None - self.assertEqual(golden, compiled) - finally: - shutil.rmtree(tmpdir) - - def test_py_compile_basic(self): - """Test basic sequential pipeline.""" - self._test_py_compile_zip('basic') - - def test_py_compile_with_sidecar(self): - """Test pipeline with sidecar.""" - self._test_py_compile_yaml('sidecar') - - def test_py_compile_with_pipelineparams(self): - """Test pipeline with multiple pipeline params.""" - self._test_py_compile_yaml('pipelineparams') - - def test_py_compile_with_opsgroups(self): - """Test pipeline with multiple opsgroups.""" - self._test_py_compile_yaml('opsgroups') - - def test_py_compile_condition(self): - """Test a pipeline with conditions.""" - self._test_py_compile_zip('coin') - - def test_py_compile_default_value(self): - """Test a pipeline with a parameter with default value.""" - self._test_py_compile_targz('default_value') - - def test_py_volume(self): - """Test a pipeline with a volume and volume mount.""" - self._test_py_compile_yaml('volume') - - @parameterized.parameters( - { - 'mode': 'V2_COMPATIBLE', - 'error': True - }, - { - 'mode': 'V1', - 'is_v2': False - }, - { - 'mode': 'V1_LEGACY', - 'is_v2': False - }, - { - 'mode': None, - 'is_v2': False - }, - { - 'mode': 'V2_COMPATIBLE', - 'env': 'V1', - 'error': True - }, - { - 'mode': None, - 'env': 'V1', - 'is_v2': False - }, - { - 'mode': None, - 'env': 'V2_COMPATIBLE', - 'error': True - }, - { - 'mode': None, - 'env': 'V1_LEGACY', - 'is_v2': False - }, - { - 'mode': 'INVALID', - 'error': True - }, - { - 'mode': None, - 'env': 'INVALID', - 'error': True - }, - ) - def test_dsl_compile_mode(self, - mode: Optional[str] = None, - is_v2: Optional[bool] = None, - env: Optional[str] = None, - error: Optional[bool] = None): - with mock.patch.dict(os.environ, - env and {'KF_PIPELINES_COMPILER_MODE': env} or {}): - file_base_name = 'two_step' - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - py_file = os.path.join(test_data_dir, f'{file_base_name}.py') - tmpdir = tempfile.mkdtemp() - try: - target_yaml = os.path.join(tmpdir, f'{file_base_name}.yaml') - args = ['dsl-compile', '--py', py_file, '--output', target_yaml] - if mode: - args = args + ['--mode', mode] - got_error = None - compiled = None - try: - subprocess.check_output(args) - with open(target_yaml, 'r') as f: - compiled = yaml.safe_load(f) - except subprocess.CalledProcessError as err: - got_error = err - if error: - if not got_error: - self.fail(f'expected error, but succeeded') - else: - if got_error: - self.fail(f'expected success, but got {got_error}') - v2_pipeline_annotation = compiled['metadata'][ - 'annotations'].get('pipelines.kubeflow.org/v2_pipeline') - if is_v2: - self.assertEqual( - 'true', v2_pipeline_annotation, - f'expected to compile in v2_compatible mode') - else: - self.assertEqual(None, v2_pipeline_annotation, - f'expected to compile in v1 mode') - finally: - shutil.rmtree(tmpdir) - - def test_py_retry_policy(self): - """Test retry policy is set.""" - - policy = 'Always' - backoff_duration = '2m' - backoff_factor = 1.5 - backoff_max_duration = '3m' - - def my_pipeline(): - some_op().set_retry(2, policy, backoff_duration, backoff_factor, - backoff_max_duration) - - workflow = kfp.compiler.Compiler()._compile(my_pipeline) - name_to_template = { - template['name']: template - for template in workflow['spec']['templates'] - } - main_dag_tasks = name_to_template[workflow['spec'] - ['entrypoint']]['dag']['tasks'] - template = name_to_template[main_dag_tasks[0]['template']] - - self.assertEqual(template['retryStrategy']['retryPolicy'], policy) - self.assertEqual(template['retryStrategy']['backoff']['duration'], - backoff_duration) - self.assertEqual(template['retryStrategy']['backoff']['factor'], - backoff_factor) - self.assertEqual(template['retryStrategy']['backoff']['maxDuration'], - backoff_max_duration) - - def test_py_runtime_memory_request(self): - """Test memory request.""" - - def my_pipeline(memory: str, cpu: str): - some_op().set_cpu_request(memory) - - workflow = kfp.compiler.Compiler()._create_workflow(my_pipeline) - name_to_template = { - template['name']: template - for template in workflow['spec']['templates'] - } - main_dag_tasks = name_to_template[workflow['spec'] - ['entrypoint']]['dag']['tasks'] - template = name_to_template[main_dag_tasks[0]['template']] - - self.assertEqual( - template['podSpecPatch'], - '{"containers": [{"name": "main", "resources": {"requests": {"cpu": "{{inputs.parameters.memory}}"}}}]}' - ) - - def test_py_runtime_gpu_request(self): - """Test GPU request.""" - - def my_pipeline(nbr_gpus: int, gpu_vendor: str): - some_op().set_gpu_limit(nbr_gpus, gpu_vendor) - - workflow = kfp.compiler.Compiler()._create_workflow(my_pipeline) - name_to_template = { - template['name']: template - for template in workflow['spec']['templates'] - } - main_dag_tasks = name_to_template[workflow['spec'] - ['entrypoint']]['dag']['tasks'] - template = name_to_template[main_dag_tasks[0]['template']] - - self.assertEqual( - template['podSpecPatch'], - '{"containers": [{"name": "main", "resources": {"limits": {"{{inputs.parameters.gpu_vendor}}": "{{inputs.parameters.nbr_gpus}}"}}}]}' - ) - - def test_py_runtime_node_selection(self): - """Test node selection request.""" - - def my_pipeline(constrain_type: str, constrain_value: str): - some_op().add_node_selector_constraint(constrain_type, - constrain_value) - - workflow = kfp.compiler.Compiler()._create_workflow(my_pipeline) - name_to_template = { - template['name']: template - for template in workflow['spec']['templates'] - } - main_dag_tasks = name_to_template[workflow['spec'] - ['entrypoint']]['dag']['tasks'] - template = name_to_template[main_dag_tasks[0]['template']] - - self.assertEqual( - template['podSpecPatch'], - '{"nodeSelector": {"{{inputs.parameters.constrain_type}}": "{{inputs.parameters.constrain_value}}"}}' - ) - - def test_py_retry_policy_invalid(self): - - def my_pipeline(): - some_op().set_retry(2, 'Invalid') - - with self.assertRaises(ValueError): - kfp.compiler.Compiler()._compile(my_pipeline) - - def test_py_retry(self): - """Test retry functionality.""" - number_of_retries = 137 - - def my_pipeline(): - some_op().set_retry(number_of_retries) - - workflow = kfp.compiler.Compiler()._compile(my_pipeline) - name_to_template = { - template['name']: template - for template in workflow['spec']['templates'] - } - main_dag_tasks = name_to_template[workflow['spec'] - ['entrypoint']]['dag']['tasks'] - template = name_to_template[main_dag_tasks[0]['template']] - - self.assertEqual(template['retryStrategy']['limit'], number_of_retries) - - def test_affinity(self): - """Test affinity functionality.""" - exp_affinity = { - 'affinity': { - 'nodeAffinity': { - 'requiredDuringSchedulingIgnoredDuringExecution': { - 'nodeSelectorTerms': [{ - 'matchExpressions': [{ - 'key': 'beta.kubernetes.io/instance-type', - 'operator': 'In', - 'values': ['p2.xlarge'] - }] - }] - } - } - } - } - - def my_pipeline(): - affinity = V1Affinity( - node_affinity=V1NodeAffinity( - required_during_scheduling_ignored_during_execution=V1NodeSelector( - node_selector_terms=[ - V1NodeSelectorTerm(match_expressions=[ - V1NodeSelectorRequirement( - key='beta.kubernetes.io/instance-type', - operator='In', - values=['p2.xlarge']) - ]) - ]))) - some_op().add_affinity(affinity) - - workflow = kfp.compiler.Compiler()._compile(my_pipeline) - - self.assertEqual(workflow['spec']['templates'][1]['affinity'], - exp_affinity['affinity']) - - def test_py_image_pull_secrets(self): - """Test pipeline imagepullsecret.""" - self._test_sample_py_compile_yaml('imagepullsecrets') - - def test_py_timeout(self): - """Test pipeline timeout.""" - self._test_py_compile_yaml('timeout') - - def test_py_recursive_do_while(self): - """Test pipeline recursive.""" - self._test_py_compile_yaml('recursive_do_while') - - def test_py_recursive_while(self): - """Test pipeline recursive.""" - self._test_py_compile_yaml('recursive_while') - - def test_py_resourceop_basic(self): - """Test pipeline resourceop_basic.""" - self._test_py_compile_yaml('resourceop_basic') - - def test_py_volumeop_basic(self): - """Test pipeline volumeop_basic.""" - self._test_py_compile_yaml('volumeop_basic') - - def test_py_volumeop_parallel(self): - """Test pipeline volumeop_parallel.""" - self._test_py_compile_yaml('volumeop_parallel') - - def test_py_volumeop_dag(self): - """Test pipeline volumeop_dag.""" - self._test_py_compile_yaml('volumeop_dag') - - def test_py_volume_snapshotop_sequential(self): - """Test pipeline volume_snapshotop_sequential.""" - self._test_py_compile_yaml('volume_snapshotop_sequential') - - def test_py_volume_snapshotop_rokurl(self): - """Test pipeline volumeop_sequential.""" - self._test_py_compile_yaml('volume_snapshotop_rokurl') - - def test_py_volumeop_sequential(self): - """Test pipeline volumeop_sequential.""" - self._test_py_compile_yaml('volumeop_sequential') - - def test_py_param_substitutions(self): - """Test pipeline param_substitutions.""" - self._test_py_compile_yaml('param_substitutions') - - def test_py_param_op_transform(self): - """Test pipeline param_op_transform.""" - self._test_py_compile_yaml('param_op_transform') - - def test_py_preemptible_gpu(self): - """Test preemptible GPU/TPU sample.""" - self._test_sample_py_compile_yaml('preemptible_tpu_gpu') - - def test_type_checking_with_consistent_types(self): - """Test type check pipeline parameters against component metadata.""" - - @component - def a_op( - field_m: {'GCSPath': { - 'path_type': 'file', - 'file_type': 'tsv' - }}, field_o: Integer()): - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_m, - '--field-o', - field_o, - ], - ) - - @pipeline(name='p1', description='description1') - def my_pipeline( - a: {'GCSPath': { - 'path_type': 'file', - 'file_type': 'tsv' - }} = 'good', - b: Integer() = 12): - a_op(field_m=a, field_o=b) - - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - sys.path.append(test_data_dir) - tmpdir = tempfile.mkdtemp() - try: - simple_package_path = os.path.join(tmpdir, 'simple.tar.gz') - compiler.Compiler().compile( - my_pipeline, simple_package_path, type_check=True) - - finally: - shutil.rmtree(tmpdir) - - def test_type_checking_with_inconsistent_types(self): - """Test type check pipeline parameters against component metadata.""" - - @component - def a_op( - field_m: {'GCSPath': { - 'path_type': 'file', - 'file_type': 'tsv' - }}, field_o: Integer()): - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_m, - '--field-o', - field_o, - ], - ) - - @pipeline(name='p1', description='description1') - def my_pipeline( - a: {'GCSPath': { - 'path_type': 'file', - 'file_type': 'csv' - }} = 'good', - b: Integer() = 12): - a_op(field_m=a, field_o=b) - - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - sys.path.append(test_data_dir) - tmpdir = tempfile.mkdtemp() - try: - simple_package_path = os.path.join(tmpdir, 'simple.tar.gz') - with self.assertRaises(InconsistentTypeException): - compiler.Compiler().compile( - my_pipeline, simple_package_path, type_check=True) - compiler.Compiler().compile( - my_pipeline, simple_package_path, type_check=False) - - finally: - shutil.rmtree(tmpdir) - - def test_type_checking_with_json_schema(self): - """Test type check pipeline parameters against the json schema.""" - - @component - def a_op( - field_m: { - 'GCRPath': { - 'openapi_schema_validator': { - "type": "string", - "pattern": "^.*gcr\\.io/.*$" - } - } - }, field_o: 'Integer'): - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_m, - '--field-o', - field_o, - ], - ) - - @pipeline(name='p1', description='description1') - def my_pipeline( - a: { - 'GCRPath': { - 'openapi_schema_validator': { - "type": "string", - "pattern": "^.*gcr\\.io/.*$" - } - } - } = 'good', - b: 'Integer' = 12): - a_op(field_m=a, field_o=b) - - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - sys.path.append(test_data_dir) - tmpdir = tempfile.mkdtemp() - try: - simple_package_path = os.path.join(tmpdir, 'simple.tar.gz') - import jsonschema - with self.assertRaises(jsonschema.exceptions.ValidationError): - compiler.Compiler().compile( - my_pipeline, simple_package_path, type_check=True) - - finally: - shutil.rmtree(tmpdir) - - def test_compile_pipeline_with_after(self): - - def op(): - return dsl.ContainerOp(name='Some component name', image='image') - - @dsl.pipeline(name='Pipeline') - def pipeline(): - task1 = op() - task2 = op().after(task1) - - compiler.Compiler()._compile(pipeline) - - def _test_op_to_template_yaml(self, ops, file_base_name): - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - target_yaml = os.path.join(test_data_dir, file_base_name + '.yaml') - with open(target_yaml, 'r') as f: - expected = yaml.safe_load(f)['spec']['templates'][0] - - compiled_template = compiler._op_to_template._op_to_template(ops) - - del compiled_template['name'], expected['name'] - for output in compiled_template['outputs'].get( - 'parameters', []) + compiled_template['outputs'].get( - 'artifacts', []) + expected['outputs'].get( - 'parameters', []) + expected['outputs'].get( - 'artifacts', []): - del output['name'] - - del compiled_template['metadata'] - assert compiled_template == expected - - def test_tolerations(self): - """Test a pipeline with a tolerations.""" - op1 = dsl.ContainerOp( - name='download', - image='busybox', - command=['sh', '-c'], - arguments=['sleep 10; wget localhost:5678 -O /tmp/results.txt'], - file_outputs={'downloaded': '/tmp/results.txt'}) \ - .add_toleration(V1Toleration( - effect='NoSchedule', - key='gpu', - operator='Equal', - value='run')) - - self._test_op_to_template_yaml(op1, file_base_name='tolerations') - - def test_set_display_name(self): - """Test a pipeline with a customized task names.""" - - import kfp.deprecated as kfp - op1 = kfp.components.load_component_from_text(''' -name: Component name -implementation: - container: - image: busybox -''') - - @dsl.pipeline() - def some_pipeline(): - op1().set_display_name('Custom name') - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - template = workflow_dict['spec']['templates'][0] - self.assertEqual( - template['metadata']['annotations'] - ['pipelines.kubeflow.org/task_display_name'], 'Custom name') - - def test_set_dynamic_display_name(self): - """Test a pipeline with a customized task names.""" - - def some_pipeline(custom_name): - some_op().set_display_name(custom_name) - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - template = [ - template for template in workflow_dict['spec']['templates'] - if 'container' in template - ][0] - self.assertNotIn( - 'pipelineparam', template['metadata']['annotations'] - ['pipelines.kubeflow.org/task_display_name']) - - def test_set_parallelism(self): - """Test a pipeline with parallelism limits.""" - - def some_op(): - return dsl.ContainerOp( - name='sleep', - image='busybox', - command=['sleep 1'], - ) - - @dsl.pipeline() - def some_pipeline(): - some_op() - some_op() - some_op() - dsl.get_pipeline_conf().set_parallelism(1) - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - self.assertEqual(workflow_dict['spec']['parallelism'], 1) - - def test_set_ttl_seconds_after_finished(self): - """Test a pipeline with ttl after finished.""" - - def some_op(): - return dsl.ContainerOp( - name='sleep', - image='busybox', - command=['sleep 1'], - ) - - @dsl.pipeline() - def some_pipeline(): - some_op() - dsl.get_pipeline_conf().set_ttl_seconds_after_finished(86400) - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - self.assertEqual(workflow_dict['spec']['ttlStrategy']['secondsAfterCompletion'], - 86400) - - def test_pod_disruption_budget(self): - """Test a pipeline with poddisruption budget.""" - - def some_op(): - return dsl.ContainerOp( - name='sleep', - image='busybox', - command=['sleep 1'], - ) - - @dsl.pipeline() - def some_pipeline(): - some_op() - dsl.get_pipeline_conf().set_pod_disruption_budget("100%") - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - self.assertEqual( - workflow_dict['spec']["podDisruptionBudget"]['minAvailable'], - "100%") - - def test_op_transformers(self): - - def some_op(): - return dsl.ContainerOp( - name='sleep', - image='busybox', - command=['sleep 1'], - ) - - @dsl.pipeline(name='some_pipeline') - def some_pipeline(): - task1 = some_op() - task2 = some_op() - task3 = some_op() - - dsl.get_pipeline_conf().op_transformers.append( - lambda op: op.set_retry(5)) - - workflow_dict = compiler.Compiler()._compile(some_pipeline) - for template in workflow_dict['spec']['templates']: - container = template.get('container', None) - if container: - self.assertEqual(template['retryStrategy']['limit'], 5) - - def test_image_pull_policy(self): - - def some_op(): - return dsl.ContainerOp( - name='sleep', - image='busybox', - command=['sleep 1'], - ) - - @dsl.pipeline(name='some_pipeline') - def some_pipeline(): - task1 = some_op() - task2 = some_op() - task3 = some_op() - - dsl.get_pipeline_conf().set_image_pull_policy(policy="Always") - - workflow_dict = compiler.Compiler()._compile(some_pipeline) - for template in workflow_dict['spec']['templates']: - container = template.get('container', None) - if container: - self.assertEqual(template['container']['imagePullPolicy'], - "Always") - - def test_image_pull_policy_step_spec(self): - - def some_op(): - return dsl.ContainerOp( - name='sleep', - image='busybox', - command=['sleep 1'], - ) - - def some_other_op(): - return dsl.ContainerOp( - name='other', - image='busybox', - command=['sleep 1'], - ) - - @dsl.pipeline(name='some_pipeline') - def some_pipeline(): - task1 = some_op() - task2 = some_op() - task3 = some_other_op().set_image_pull_policy("IfNotPresent") - - dsl.get_pipeline_conf().set_image_pull_policy(policy="Always") - - workflow_dict = compiler.Compiler()._compile(some_pipeline) - for template in workflow_dict['spec']['templates']: - container = template.get('container', None) - if container: - if template['name'] == "other": - self.assertEqual(template['container']['imagePullPolicy'], - "IfNotPresent") - elif template['name'] == "sleep": - self.assertEqual(template['container']['imagePullPolicy'], - "Always") - - def test_image_pull_policy_invalid_setting(self): - - def some_op(): - return dsl.ContainerOp( - name='sleep', - image='busybox', - command=['sleep 1'], - ) - - with self.assertRaises(ValueError): - - @dsl.pipeline(name='some_pipeline') - def some_pipeline(): - task1 = some_op() - task2 = some_op() - dsl.get_pipeline_conf().set_image_pull_policy(policy="Alwayss") - - workflow_dict = compiler.Compiler()._compile(some_pipeline) - - def test_set_default_pod_node_selector(self): - """Test a pipeline with node selector.""" - - def some_op(): - return dsl.ContainerOp( - name='sleep', - image='busybox', - command=['sleep 1'], - ) - - @dsl.pipeline() - def some_pipeline(): - some_op() - dsl.get_pipeline_conf().set_default_pod_node_selector( - label_name="cloud.google.com/gke-accelerator", - value="nvidia-tesla-p4") - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - self.assertEqual( - workflow_dict['spec']['nodeSelector'], - {"cloud.google.com/gke-accelerator": "nvidia-tesla-p4"}) - - def test_set_dns_config(self): - """Test a pipeline with node selector.""" - - @dsl.pipeline() - def some_pipeline(): - some_op() - dsl.get_pipeline_conf().set_dns_config( - V1PodDNSConfig( - nameservers=["1.2.3.4"], - options=[V1PodDNSConfigOption(name="ndots", value="2")])) - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - self.assertEqual( - workflow_dict['spec']['dnsConfig'], { - "nameservers": ["1.2.3.4"], - "options": [{ - "name": "ndots", - "value": "2" - }] - }) - - def test_container_op_output_error_when_no_or_multiple_outputs(self): - - def no_outputs_pipeline(): - no_outputs_op = dsl.ContainerOp(name='dummy', image='dummy') - dsl.ContainerOp( - name='dummy', image='dummy', arguments=[no_outputs_op.output]) - - def one_output_pipeline(): - one_output_op = dsl.ContainerOp( - name='dummy', image='dummy', file_outputs={'out1': 'path1'}) - dsl.ContainerOp( - name='dummy', image='dummy', arguments=[one_output_op.output]) - - def two_outputs_pipeline(): - two_outputs_op = dsl.ContainerOp( - name='dummy', - image='dummy', - file_outputs={ - 'out1': 'path1', - 'out2': 'path2' - }) - dsl.ContainerOp( - name='dummy', image='dummy', arguments=[two_outputs_op.output]) - - with self.assertRaises(RuntimeError): - compiler.Compiler()._compile(no_outputs_pipeline) - - compiler.Compiler()._compile(one_output_pipeline) - - with self.assertRaises(RuntimeError): - compiler.Compiler()._compile(two_outputs_pipeline) - - def test_withitem_basic(self): - self._test_py_compile_yaml('withitem_basic') - - def test_withitem_nested(self): - self._test_py_compile_yaml('withitem_nested') - - def test_add_pod_env(self): - self._test_py_compile_yaml('add_pod_env') - - def test_init_container(self): - echo = dsl.UserContainer( - name='echo', image='alpine:latest', command=['echo', 'bye']) - - @dsl.pipeline( - name='InitContainer', description='A pipeline with init container.') - def init_container_pipeline(): - dsl.ContainerOp( - name='hello', - image='alpine:latest', - command=['echo', 'hello'], - init_containers=[echo]) - - workflow_dict = compiler.Compiler()._compile(init_container_pipeline) - for template in workflow_dict['spec']['templates']: - init_containers = template.get('initContainers', None) - if init_containers: - self.assertEqual(len(init_containers), 1) - init_container = init_containers[0] - self.assertEqual( - init_container, { - 'image': 'alpine:latest', - 'command': ['echo', 'bye'], - 'name': 'echo' - }) - - def test_delete_resource_op(self): - """Test a pipeline with a delete resource operation.""" - from kubernetes import client as k8s - - @dsl.pipeline() - def some_pipeline(): - # create config map object with k6 load test script - config_map = k8s.V1ConfigMap( - api_version="v1", - data={"foo": "bar"}, - kind="ConfigMap", - metadata=k8s.V1ObjectMeta( - name="foo-bar-cm", namespace="default")) - # delete the config map in k8s - dsl.ResourceOp( - name="delete-config-map", - action="delete", - k8s_resource=config_map) - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - delete_op_template = [ - template for template in workflow_dict['spec']['templates'] - if template['name'] == 'delete-config-map' - ][0] - - # delete resource operation should not have success condition, failure condition or output parameters. - # See https://github.com/argoproj/argo-workflows/blob/5331fc02e257266a4a5887dfe6277e5a0b42e7fc/cmd/argoexec/commands/resource.go#L30 - self.assertIsNone(delete_op_template.get("successCondition")) - self.assertIsNone(delete_op_template.get("failureCondition")) - self.assertDictEqual(delete_op_template.get("outputs", {}), {}) - - def test_withparam_global(self): - self._test_py_compile_yaml('withparam_global') - - def test_withparam_global_dict(self): - self._test_py_compile_yaml('withparam_global_dict') - - def test_withparam_output(self): - self._test_py_compile_yaml('withparam_output') - - def test_withparam_output_dict(self): - self._test_py_compile_yaml('withparam_output_dict') - - def test_withparam_lightweight_out(self): - self._test_py_compile_yaml('loop_over_lightweight_output') - - def test_parallelfor_pipeline_param_in_items_resolving(self): - self._test_py_compile_yaml( - 'parallelfor_pipeline_param_in_items_resolving') - - def test_parallelfor_item_argument_resolving(self): - self._test_py_compile_yaml('parallelfor_item_argument_resolving') - - def test_py_input_artifact_raw_value(self): - """Test pipeline input_artifact_raw_value.""" - self._test_py_compile_yaml('input_artifact_raw_value') - - def test_pipeline_name_same_as_task_name(self): - - def some_name(): - dsl.ContainerOp( - name='some_name', - image='alpine:latest', - ) - - workflow_dict = compiler.Compiler()._compile(some_name) - template_names = set( - template['name'] for template in workflow_dict['spec']['templates']) - self.assertGreater(len(template_names), 1) - self.assertEqual(template_names, {'some-name', 'some-name-2'}) - - def test_set_execution_options_caching_strategy(self): - - def some_pipeline(): - task = some_op() - task.execution_options.caching_strategy.max_cache_staleness = "P30D" - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - template = workflow_dict['spec']['templates'][0] - self.assertEqual( - template['metadata']['annotations'] - ['pipelines.kubeflow.org/max_cache_staleness'], "P30D") - - def test_artifact_passing_using_volume(self): - self._test_py_compile_yaml('artifact_passing_using_volume') - - def test_recursive_argument_mapping(self): - # Verifying that the recursive call arguments are passed correctly when specified out of order - component_2_in_0_out_op = kfp.components.load_component_from_text(''' -inputs: -- name: in1 -- name: in2 -implementation: - container: - image: busybox - command: - - echo - - inputValue: in1 - - inputValue: in2 - ''') - - @dsl.graph_component - def subgraph(graph_in1, graph_in2): - component_2_in_0_out_op( - in1=graph_in1, - in2=graph_in2, - ) - subgraph( - # Wrong order! - graph_in2=graph_in2, - graph_in1=graph_in1, - ) - - def some_pipeline(pipeline_in1, pipeline_in2): - subgraph(pipeline_in1, pipeline_in2) - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - subgraph_template = [ - template for template in workflow_dict['spec']['templates'] - if 'subgraph' in template['name'] - ][0] - recursive_subgraph_task = [ - task for task in subgraph_template['dag']['tasks'] - if 'subgraph' in task['name'] - ][0] - for argument in recursive_subgraph_task['arguments']['parameters']: - if argument['name'].endswith('in1'): - self.assertTrue( - argument['value'].endswith('in1}}'), - 'Wrong argument mapping: "{}" passed to "{}"'.format( - argument['value'], argument['name'])) - elif argument['name'].endswith('in2'): - self.assertTrue( - argument['value'].endswith('in2}}'), - 'Wrong argument mapping: "{}" passed to "{}"'.format( - argument['value'], argument['name'])) - else: - self.fail('Unexpected input name: ' + argument['name']) - - def test_input_name_sanitization(self): - component_2_in_1_out_op = kfp.components.load_component_from_text(''' -inputs: -- name: Input 1 -- name: Input 2 -outputs: -- name: Output 1 -implementation: - container: - image: busybox - command: - - echo - - inputValue: Input 1 - - inputPath: Input 2 - - outputPath: Output 1 - ''') - - def some_pipeline(): - task1 = component_2_in_1_out_op('value 1', 'value 2') - component_2_in_1_out_op(task1.output, task1.output) - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - container_templates = [ - template for template in workflow_dict['spec']['templates'] - if 'container' in template - ] - for template in container_templates: - for argument in template['inputs'].get('parameters', []): - self.assertNotIn( - ' ', argument['name'], - 'The input name "{}" of template "{}" was not sanitized.' - .format(argument['name'], template['name'])) - for argument in template['inputs']['artifacts']: - self.assertNotIn( - ' ', argument['name'], - 'The input name "{}" of template "{}" was not sanitized.' - .format(argument['name'], template['name'])) - - def test_container_op_with_arbitrary_name(self): - - def some_pipeline(): - dsl.ContainerOp( - name=r''' !"#$%&'()*+,-./:;<=>?@[\]^_`''', - image='alpine:latest', - ) - dsl.ContainerOp( - name=r''' !"#$%&'()*+,-./:;<=>?@[\]^_`''', - image='alpine:latest', - ) - - workflow_dict = compiler.Compiler()._compile(some_pipeline) - for template in workflow_dict['spec']['templates']: - self.assertNotEqual(template['name'], '') - - def test_empty_string_pipeline_parameter_defaults(self): - - def some_pipeline(param1: str = ''): - pass - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - self.assertEqual( - workflow_dict['spec']['arguments']['parameters'][0].get('value'), - '') - - def test_preserving_parameter_arguments_map(self): - component_2_in_1_out_op = kfp.components.load_component_from_text(''' -inputs: -- name: Input 1 -- name: Input 2 -outputs: -- name: Output 1 -implementation: - container: - image: busybox - command: - - echo - - inputValue: Input 1 - - inputPath: Input 2 - - outputPath: Output 1 - ''') - - def some_pipeline(): - task1 = component_2_in_1_out_op('value 1', 'value 2') - component_2_in_1_out_op(task1.output, task1.output) - - workflow_dict = kfp.compiler.Compiler()._compile(some_pipeline) - container_templates = [ - template for template in workflow_dict['spec']['templates'] - if 'container' in template - ] - for template in container_templates: - parameter_arguments_json = template['metadata']['annotations'][ - 'pipelines.kubeflow.org/arguments.parameters'] - parameter_arguments = json.loads(parameter_arguments_json) - self.assertEqual(set(parameter_arguments.keys()), {'Input 1'}) - - def test__resolve_task_pipeline_param(self): - p = PipelineParam(name='param2') - resolved = Compiler._resolve_task_pipeline_param(p, group_type=None) - self.assertEqual(resolved, "{{workflow.parameters.param2}}") - - p = PipelineParam(name='param1', op_name='op1') - resolved = Compiler._resolve_task_pipeline_param(p, group_type=None) - self.assertEqual(resolved, - "{{tasks.op1.outputs.parameters.op1-param1}}") - - p = PipelineParam(name='param1', op_name='op1') - resolved = Compiler._resolve_task_pipeline_param( - p, group_type="subgraph") - self.assertEqual(resolved, "{{inputs.parameters.op1-param1}}") - - @unittest.skip('v2 compatible mode is being deprecated in SDK v2.0') - def test_uri_artifact_passing(self): - self._test_py_compile_yaml('uri_artifacts', mode='V2_COMPATIBLE') - - def test_keyword_only_argument_for_pipeline_func(self): - - def some_pipeline(casual_argument: str, *, keyword_only_argument: str): - pass - - kfp.compiler.Compiler()._create_workflow(some_pipeline) - - def test_keyword_only_argument_for_pipeline_func_identity(self): - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - sys.path.append(test_data_dir) - - # `@pipeline` is needed to make name the same for both functions - - @pipeline(name="pipeline_func") - def pipeline_func_arg(foo_arg: str, bar_arg: str): - dsl.ContainerOp( - name='foo', - image='foo', - command=['bar'], - arguments=[foo_arg, ' and ', bar_arg]) - - @pipeline(name="pipeline_func") - def pipeline_func_kwarg(foo_arg: str, *, bar_arg: str): - return pipeline_func_arg(foo_arg, bar_arg) - - pipeline_yaml_arg = kfp.compiler.Compiler()._create_workflow( - pipeline_func_arg) - pipeline_yaml_kwarg = kfp.compiler.Compiler()._create_workflow( - pipeline_func_kwarg) - - # the yamls may differ in metadata - def remove_metadata(yaml) -> None: - del yaml['metadata'] - - remove_metadata(pipeline_yaml_arg) - remove_metadata(pipeline_yaml_kwarg) - - # compare - self.assertEqual(pipeline_yaml_arg, pipeline_yaml_kwarg) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/tests/compiler/container_builder_test.py b/sdk/python/tests/compiler/container_builder_test.py index 98c8e48a3bf..5fd4b20e591 100644 --- a/sdk/python/tests/compiler/container_builder_test.py +++ b/sdk/python/tests/compiler/container_builder_test.py @@ -24,7 +24,7 @@ DEFAULT_IMAGE_NAME = 'gcr.io/kfp-testing/image' -@mock.patch('kfp.containers._gcs_helper.GCSHelper') +@mock.patch('kfp.deprecated.containers._gcs_helper.GCSHelper') class TestContainerBuild(unittest.TestCase): def test_wrap_dir_in_tarball(self, mock_gcshelper): From 6a98d98defeb8f6c46436af8c8123109054aabc3 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Sun, 13 Feb 2022 00:01:00 -0800 Subject: [PATCH 09/39] retore v2 test changes --- .../test_data/lightweight_python_functions_v2_pipeline.json | 2 +- .../test_data/lightweight_python_functions_v2_with_outputs.json | 2 +- .../v2/compiler_cli_tests/test_data/pipeline_with_after.json | 2 +- .../test_data/pipeline_with_concat_placeholder.json | 2 +- .../compiler_cli_tests/test_data/pipeline_with_condition.json | 2 +- .../kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json | 2 +- .../test_data/pipeline_with_exit_handler.json | 2 +- .../compiler_cli_tests/test_data/pipeline_with_gcpc_types.json | 2 +- .../test_data/pipeline_with_if_placeholder.json | 2 +- .../v2/compiler_cli_tests/test_data/pipeline_with_importer.json | 2 +- .../v2/compiler_cli_tests/test_data/pipeline_with_loops.json | 2 +- .../test_data/pipeline_with_loops_and_conditions.json | 2 +- .../test_data/pipeline_with_metrics_outputs.json | 2 +- .../test_data/pipeline_with_nested_conditions.json | 2 +- .../test_data/pipeline_with_nested_conditions_yaml.json | 2 +- .../test_data/pipeline_with_nested_loops.json | 2 +- .../v2/compiler_cli_tests/test_data/pipeline_with_ontology.json | 2 +- .../test_data/pipeline_with_params_containing_format.json | 2 +- .../test_data/pipeline_with_placeholders.json | 2 +- .../test_data/pipeline_with_resource_spec.json | 2 +- .../test_data/pipeline_with_reused_component.json | 2 +- .../test_data/pipeline_with_various_io_types.json | 2 +- .../kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json | 2 +- .../test_data/v2_component_with_optional_inputs.json | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json index 3a4f1b620be..822cf38cb3e 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_pipeline.json @@ -245,5 +245,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json index a62d96495c7..f76b13e5b9f 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/lightweight_python_functions_v2_with_outputs.json @@ -328,5 +328,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json index 7b528973fb6..61b1c6af868 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_after.json @@ -146,5 +146,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json index a8588ccfcef..780ba6bc193 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_concat_placeholder.json @@ -55,5 +55,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json index cad42d7bf72..fb1dee75633 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_condition.json @@ -301,5 +301,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json index 29230e105a9..b2df9240e38 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_env.json @@ -90,5 +90,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json index 0287adb21b1..34d8770d503 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_exit_handler.json @@ -205,5 +205,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json index e223f96e826..f5255ec95ec 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_gcpc_types.json @@ -107,5 +107,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json index 222eb9c1874..32665fd1e22 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_if_placeholder.json @@ -76,5 +76,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json index 8cd7a589bee..6d94c264f82 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_importer.json @@ -315,5 +315,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json index 1d157072c4c..98c21914203 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops.json @@ -510,5 +510,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json index cb478caca27..f2d831afdc5 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_loops_and_conditions.json @@ -1292,5 +1292,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json index d7e8cac3e5d..ed8631ab6ea 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_metrics_outputs.json @@ -189,5 +189,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json index 8cd4d42c0b5..530b2a8499c 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions.json @@ -491,5 +491,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json index 78fc3594abf..e419c688cb7 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_conditions_yaml.json @@ -536,5 +536,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json index 5f0ff0ad4db..f348e3d3fc6 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_nested_loops.json @@ -342,5 +342,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json index 4efd952a14d..c97f55c7b60 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_ontology.json @@ -159,5 +159,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json index 3a260927888..f66b85256e3 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_params_containing_format.json @@ -251,5 +251,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_placeholders.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_placeholders.json index 2e764731551..e9637476d17 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_placeholders.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_placeholders.json @@ -305,5 +305,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json index 80182293c30..a6aacf502ee 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_resource_spec.json @@ -167,5 +167,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json index ce95f2d0c16..1fe025cd751 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_reused_component.json @@ -206,5 +206,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json index 855d23b03ed..5b4eeda5f05 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_various_io_types.json @@ -303,5 +303,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json index 6b78e984a0d..63628409903 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/two_step_pipeline.json @@ -122,5 +122,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json index fcef5d3b738..1ad469ff484 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/v2_component_with_optional_inputs.json @@ -73,5 +73,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file From d5e516099b504e8c8f32269e17651c1ca393e7f5 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Sun, 13 Feb 2022 00:33:15 -0800 Subject: [PATCH 10/39] fix py3.6 test --- sdk/python/kfp/deprecated/dsl/_container_op.py | 10 +++++----- .../test_data/xgboost_sample_pipeline.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/kfp/deprecated/dsl/_container_op.py b/sdk/python/kfp/deprecated/dsl/_container_op.py index b156dd5f4b2..e0f39c4a862 100644 --- a/sdk/python/kfp/deprecated/dsl/_container_op.py +++ b/sdk/python/kfp/deprecated/dsl/_container_op.py @@ -17,7 +17,7 @@ from typing import (Any, Callable, Dict, List, Optional, Sequence, Tuple, TypeVar, Union) -import kfp.deprecated as kfp +from kfp.deprecated._config import COMPILING_FOR_V2 from kfp.deprecated.components import _components, _structures from kfp.deprecated.dsl import _pipeline_param, dsl_utils from kfp.pipeline_spec import pipeline_spec_pb2 @@ -467,7 +467,7 @@ def set_env_variable(self, name: str, value: str) -> 'Container': value: The value of the environment variable. """ - if not kfp.COMPILING_FOR_V2: + if not COMPILING_FOR_V2: raise ValueError( 'set_env_variable is v2 only. Use add_env_variable for v1.') @@ -1264,7 +1264,7 @@ def resolve_artifact_argument(artarg): 'https://kubeflow-pipelines.readthedocs.io/en/stable/source/kfp.components.html#kfp.components.load_component_from_file', category=FutureWarning, ) - if kfp.COMPILING_FOR_V2: + if COMPILING_FOR_V2: raise RuntimeError( 'Constructing ContainerOp instances directly is deprecated and not ' 'supported when compiling to v2 (using v2 compiler or v1 compiler ' @@ -1311,7 +1311,7 @@ def _decorated(*args, **kwargs): 'outputting big data.', DeprecationWarning) # Skip the special handling that is unnecessary in v2. - if not kfp.COMPILING_FOR_V2: + if not COMPILING_FOR_V2: # Special handling for the mlpipeline-ui-metadata and mlpipeline-metrics # outputs that should always be saved as artifacts # TODO: Remove when outputs are always saved as artifacts @@ -1452,7 +1452,7 @@ def _set_metadata(self, output.name) self.file_outputs[output.name] = output_filename - if not kfp.COMPILING_FOR_V2: + if not COMPILING_FOR_V2: for output_name, path in dict(self.file_outputs).items(): is_legacy_name, normalized_name = _is_legacy_output_name( output_name) diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json index 57129abf9b0..df118cf6a5e 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/xgboost_sample_pipeline.json @@ -755,5 +755,5 @@ } }, "schemaVersion": "2.1.0", - "sdkVersion": "kfp-1.8.11" + "sdkVersion": "kfp-1.8.9" } \ No newline at end of file From 232567b198d645a0ac40e0daf3898cebd6435daf Mon Sep 17 00:00:00 2001 From: Yaqi Date: Sun, 13 Feb 2022 19:33:38 -0800 Subject: [PATCH 11/39] fix py3.6 test --- .../kfp/deprecated/compiler/compiler.py | 14 ++++++------ sdk/python/kfp/deprecated/dsl/_component.py | 4 ++-- .../kfp/deprecated/dsl/_component_bridge.py | 22 +++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sdk/python/kfp/deprecated/compiler/compiler.py b/sdk/python/kfp/deprecated/compiler/compiler.py index 0f983022ad2..8cd571dadb7 100644 --- a/sdk/python/kfp/deprecated/compiler/compiler.py +++ b/sdk/python/kfp/deprecated/compiler/compiler.py @@ -1163,23 +1163,23 @@ def compile(self, self._pipeline_name_param = dsl.PipelineParam( name='pipeline-name', value=f'pipeline/{pipeline_name}') - import kfp.deprecated as kfp - type_check_old_value = kfp.TYPE_CHECK - compiling_for_v2_old_value = kfp.COMPILING_FOR_V2 - kfp.COMPILING_FOR_V2 = self._mode in [ + from kfp.deprecated._config import TYPE_CHECK, COMPILING_FOR_V2 + type_check_old_value = TYPE_CHECK + compiling_for_v2_old_value = COMPILING_FOR_V2 + COMPILING_FOR_V2 = self._mode in [ dsl.PipelineExecutionMode.V2_COMPATIBLE, dsl.PipelineExecutionMode.V2_ENGINE, ] try: - kfp.TYPE_CHECK = type_check + TYPE_CHECK = type_check self._create_and_write_workflow( pipeline_func=pipeline_func, pipeline_conf=pipeline_conf, package_path=package_path) finally: - kfp.TYPE_CHECK = type_check_old_value - kfp.COMPILING_FOR_V2 = compiling_for_v2_old_value + TYPE_CHECK = type_check_old_value + COMPILING_FOR_V2 = compiling_for_v2_old_value @staticmethod def _write_workflow(workflow: Dict[Text, Any], package_path: Text = None): diff --git a/sdk/python/kfp/deprecated/dsl/_component.py b/sdk/python/kfp/deprecated/dsl/_component.py index b490ad51f06..1449a2df060 100644 --- a/sdk/python/kfp/deprecated/dsl/_component.py +++ b/sdk/python/kfp/deprecated/dsl/_component.py @@ -17,7 +17,7 @@ from ._pipeline_param import PipelineParam from .types import check_types, InconsistentTypeException from ._ops_group import Graph -import kfp.deprecated as kfp +from kfp.deprecated._config import TYPE_CHECK @deprecated( @@ -88,7 +88,7 @@ def foobar(model: TFModel(), step: MLStep()): def _component(*args, **kargs): from ..components._python_op import _extract_component_interface component_meta = _extract_component_interface(func) - if kfp.TYPE_CHECK: + if TYPE_CHECK: arg_index = 0 for arg in args: if isinstance(arg, PipelineParam) and not check_types( diff --git a/sdk/python/kfp/deprecated/dsl/_component_bridge.py b/sdk/python/kfp/deprecated/dsl/_component_bridge.py index eda811b3276..730d253c819 100644 --- a/sdk/python/kfp/deprecated/dsl/_component_bridge.py +++ b/sdk/python/kfp/deprecated/dsl/_component_bridge.py @@ -19,7 +19,7 @@ import pathlib from typing import Any, Mapping, Optional -import kfp.deprecated as kfp +from kfp.deprecated._config import COMPILING_FOR_V2 from kfp.deprecated.components import _structures, _data_passing from kfp.deprecated.components import _components from kfp.deprecated.components import _naming @@ -200,7 +200,7 @@ def _create_container_op_from_component_and_arguments( elif input_spec.type == 'Float': default_value = float(default_value) elif (type_utils.is_parameter_type(input_spec.type) and - kfp.COMPILING_FOR_V2): + COMPILING_FOR_V2): parameter_type = type_utils.get_parameter_type(input_spec.type) default_value = type_utils.deserialize_parameter_value( value=default_value, parameter_type=parameter_type) @@ -276,7 +276,7 @@ def _create_container_op_from_component_and_arguments( } for input in component_meta.inputs } - if kfp.COMPILING_FOR_V2: + if COMPILING_FOR_V2: for name, spec_type in name_to_spec_type.items(): if (name in original_arguments and type_utils.is_parameter_type(spec_type['type'])): @@ -382,7 +382,7 @@ def _resolve_commands_and_args_v2( } def _input_artifact_uri_placeholder(input_key: str) -> str: - if kfp.COMPILING_FOR_V2 and type_utils.is_parameter_type( + if COMPILING_FOR_V2 and type_utils.is_parameter_type( inputs_dict[input_key].type): raise TypeError( 'Input "{}" with type "{}" cannot be paired with ' @@ -392,7 +392,7 @@ def _input_artifact_uri_placeholder(input_key: str) -> str: return _generate_input_uri_placeholder(input_key) def _input_artifact_path_placeholder(input_key: str) -> str: - if kfp.COMPILING_FOR_V2 and type_utils.is_parameter_type( + if COMPILING_FOR_V2 and type_utils.is_parameter_type( inputs_dict[input_key].type): raise TypeError( 'Input "{}" with type "{}" cannot be paired with ' @@ -402,7 +402,7 @@ def _input_artifact_path_placeholder(input_key: str) -> str: return "{{{{$.inputs.artifacts['{}'].path}}}}".format(input_key) def _input_parameter_placeholder(input_key: str) -> str: - if kfp.COMPILING_FOR_V2 and not type_utils.is_parameter_type( + if COMPILING_FOR_V2 and not type_utils.is_parameter_type( inputs_dict[input_key].type): raise TypeError( 'Input "{}" with type "{}" cannot be paired with ' @@ -412,7 +412,7 @@ def _input_parameter_placeholder(input_key: str) -> str: return "{{{{$.inputs.parameters['{}']}}}}".format(input_key) def _output_artifact_uri_placeholder(output_key: str) -> str: - if kfp.COMPILING_FOR_V2 and type_utils.is_parameter_type( + if COMPILING_FOR_V2 and type_utils.is_parameter_type( outputs_dict[output_key].type): raise TypeError( 'Output "{}" with type "{}" cannot be paired with ' @@ -548,7 +548,7 @@ def _resolve_ir_placeholders_v2( argument_type = 'String' pipeline_params = _pipeline_param.extract_pipelineparams_from_any( argument_value) - if pipeline_params and kfp.COMPILING_FOR_V2: + if pipeline_params and COMPILING_FOR_V2: # argument_value contains PipelineParam placeholders which needs to be # replaced. And the input needs to be added to the task spec. for param in pipeline_params: @@ -622,7 +622,7 @@ def _resolve_ir_placeholders_v2( f'ContainerOp object {input_name} was passed to component as an ' 'input argument. Pass a single output instead.') else: - if kfp.COMPILING_FOR_V2: + if COMPILING_FOR_V2: raise NotImplementedError( 'Input argument supports only the following types: ' 'PipelineParam, str, int, float, bool, dict, and list. Got: ' @@ -630,7 +630,7 @@ def _resolve_ir_placeholders_v2( argument_is_parameter_type = type_utils.is_parameter_type(argument_type) input_is_parameter_type = type_utils.is_parameter_type(input_type) - if kfp.COMPILING_FOR_V2 and (argument_is_parameter_type != + if COMPILING_FOR_V2 and (argument_is_parameter_type != input_is_parameter_type): if isinstance(argument_value, dsl.PipelineParam): param_or_value_msg = 'PipelineParam "{}"'.format( @@ -684,7 +684,7 @@ def _resolve_ir_placeholders_v2( task.task_spec = pipeline_task_spec # Override command and arguments if compiling to v2. - if kfp.COMPILING_FOR_V2: + if COMPILING_FOR_V2: task.command = resolved_cmd.command task.arguments = resolved_cmd.args From ebbc687e360414f808450e877a9615f36c64ba76 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Sun, 13 Feb 2022 20:01:00 -0800 Subject: [PATCH 12/39] fix py3.6 import fallback error --- sdk/python/kfp/deprecated/compiler/compiler.py | 14 +++++++------- sdk/python/kfp/deprecated/dsl/_component.py | 4 ++-- sdk/python/kfp/deprecated/dsl/_component_bridge.py | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/sdk/python/kfp/deprecated/compiler/compiler.py b/sdk/python/kfp/deprecated/compiler/compiler.py index 8cd571dadb7..0f983022ad2 100644 --- a/sdk/python/kfp/deprecated/compiler/compiler.py +++ b/sdk/python/kfp/deprecated/compiler/compiler.py @@ -1163,23 +1163,23 @@ def compile(self, self._pipeline_name_param = dsl.PipelineParam( name='pipeline-name', value=f'pipeline/{pipeline_name}') - from kfp.deprecated._config import TYPE_CHECK, COMPILING_FOR_V2 - type_check_old_value = TYPE_CHECK - compiling_for_v2_old_value = COMPILING_FOR_V2 - COMPILING_FOR_V2 = self._mode in [ + import kfp.deprecated as kfp + type_check_old_value = kfp.TYPE_CHECK + compiling_for_v2_old_value = kfp.COMPILING_FOR_V2 + kfp.COMPILING_FOR_V2 = self._mode in [ dsl.PipelineExecutionMode.V2_COMPATIBLE, dsl.PipelineExecutionMode.V2_ENGINE, ] try: - TYPE_CHECK = type_check + kfp.TYPE_CHECK = type_check self._create_and_write_workflow( pipeline_func=pipeline_func, pipeline_conf=pipeline_conf, package_path=package_path) finally: - TYPE_CHECK = type_check_old_value - COMPILING_FOR_V2 = compiling_for_v2_old_value + kfp.TYPE_CHECK = type_check_old_value + kfp.COMPILING_FOR_V2 = compiling_for_v2_old_value @staticmethod def _write_workflow(workflow: Dict[Text, Any], package_path: Text = None): diff --git a/sdk/python/kfp/deprecated/dsl/_component.py b/sdk/python/kfp/deprecated/dsl/_component.py index 1449a2df060..b490ad51f06 100644 --- a/sdk/python/kfp/deprecated/dsl/_component.py +++ b/sdk/python/kfp/deprecated/dsl/_component.py @@ -17,7 +17,7 @@ from ._pipeline_param import PipelineParam from .types import check_types, InconsistentTypeException from ._ops_group import Graph -from kfp.deprecated._config import TYPE_CHECK +import kfp.deprecated as kfp @deprecated( @@ -88,7 +88,7 @@ def foobar(model: TFModel(), step: MLStep()): def _component(*args, **kargs): from ..components._python_op import _extract_component_interface component_meta = _extract_component_interface(func) - if TYPE_CHECK: + if kfp.TYPE_CHECK: arg_index = 0 for arg in args: if isinstance(arg, PipelineParam) and not check_types( diff --git a/sdk/python/kfp/deprecated/dsl/_component_bridge.py b/sdk/python/kfp/deprecated/dsl/_component_bridge.py index 730d253c819..1243d627494 100644 --- a/sdk/python/kfp/deprecated/dsl/_component_bridge.py +++ b/sdk/python/kfp/deprecated/dsl/_component_bridge.py @@ -19,6 +19,7 @@ import pathlib from typing import Any, Mapping, Optional +# import kfp.deprecated as kfp from kfp.deprecated._config import COMPILING_FOR_V2 from kfp.deprecated.components import _structures, _data_passing from kfp.deprecated.components import _components From 06e95ea15e08f41f69c20688389ed27734cf9480 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Mon, 14 Feb 2022 02:28:07 -0800 Subject: [PATCH 13/39] remove deprecated --- sdk/python/kfp/__init__.py | 4 + sdk/python/kfp/deprecated/dsl/__init__.py | 2 +- sdk/python/kfp/deprecated/dsl/_component.py | 96 +-- .../kfp/deprecated/dsl/_component_bridge.py | 1 - sdk/python/tests/dsl/component_tests.py | 724 +----------------- 5 files changed, 54 insertions(+), 773 deletions(-) diff --git a/sdk/python/kfp/__init__.py b/sdk/python/kfp/__init__.py index a72b6c3e13c..33c61dae084 100644 --- a/sdk/python/kfp/__init__.py +++ b/sdk/python/kfp/__init__.py @@ -18,3 +18,7 @@ __version__ = '2.0.0b0' +TYPE_CHECK = True +# COMPILING_FOR_V2 is True when using kfp.v2.compiler or use (v1) kfp.compiler +# with V2_COMPATIBLE or V2_ENGINE mode +COMPILING_FOR_V2 = False diff --git a/sdk/python/kfp/deprecated/dsl/__init__.py b/sdk/python/kfp/deprecated/dsl/__init__.py index 705c4b2f801..5adc0414443 100644 --- a/sdk/python/kfp/deprecated/dsl/__init__.py +++ b/sdk/python/kfp/deprecated/dsl/__init__.py @@ -20,7 +20,7 @@ from ._pipeline_volume import PipelineVolume from ._volume_snapshot_op import VolumeSnapshotOp from ._ops_group import OpsGroup, ExitHandler, Condition, ParallelFor, SubGraph -from ._component import python_component, graph_component, component +from ._component import graph_component, component def importer(*args, **kwargs): diff --git a/sdk/python/kfp/deprecated/dsl/_component.py b/sdk/python/kfp/deprecated/dsl/_component.py index b490ad51f06..e104641a1d5 100644 --- a/sdk/python/kfp/deprecated/dsl/_component.py +++ b/sdk/python/kfp/deprecated/dsl/_component.py @@ -20,54 +20,54 @@ import kfp.deprecated as kfp -@deprecated( - version='0.2.6', - reason='This decorator does not seem to be used, so we deprecate it. ' - 'If you need this decorator, please create an issue at ' - 'https://github.com/kubeflow/pipelines/issues', -) -def python_component(name, - description=None, - base_image=None, - target_component_file: str = None): - """Decorator for Python component functions. - - This decorator adds the metadata to the function object itself. - - Args: - name: Human-readable name of the component - description: Optional. Description of the component - base_image: Optional. Docker container image to use as the base of the - component. Needs to have Python 3.5+ installed. - target_component_file: Optional. Local file to store the component - definition. The file can then be used for sharing. - - Returns: - The same function (with some metadata fields set). - - Example: - :: - - @dsl.python_component( - name='my awesome component', - description='Come, Let\'s play', - base_image='tensorflow/tensorflow:1.11.0-py3', - ) - def my_component(a: str, b: int) -> str: - ... - """ - - def _python_component(func): - func._component_human_name = name - if description: - func._component_description = description - if base_image: - func._component_base_image = base_image - if target_component_file: - func._component_target_component_file = target_component_file - return func - - return _python_component +# @deprecated( +# version='0.2.6', +# reason='This decorator does not seem to be used, so we deprecate it. ' +# 'If you need this decorator, please create an issue at ' +# 'https://github.com/kubeflow/pipelines/issues', +# ) +# def python_component(name, +# description=None, +# base_image=None, +# target_component_file: str = None): +# """Decorator for Python component functions. + +# This decorator adds the metadata to the function object itself. + +# Args: +# name: Human-readable name of the component +# description: Optional. Description of the component +# base_image: Optional. Docker container image to use as the base of the +# component. Needs to have Python 3.5+ installed. +# target_component_file: Optional. Local file to store the component +# definition. The file can then be used for sharing. + +# Returns: +# The same function (with some metadata fields set). + +# Example: +# :: + +# @dsl.python_component( +# name='my awesome component', +# description='Come, Let\'s play', +# base_image='tensorflow/tensorflow:1.11.0-py3', +# ) +# def my_component(a: str, b: int) -> str: +# ... +# """ + +# def _python_component(func): +# func._component_human_name = name +# if description: +# func._component_description = description +# if base_image: +# func._component_base_image = base_image +# if target_component_file: +# func._component_target_component_file = target_component_file +# return func + +# return _python_component def component(func): diff --git a/sdk/python/kfp/deprecated/dsl/_component_bridge.py b/sdk/python/kfp/deprecated/dsl/_component_bridge.py index 1243d627494..730d253c819 100644 --- a/sdk/python/kfp/deprecated/dsl/_component_bridge.py +++ b/sdk/python/kfp/deprecated/dsl/_component_bridge.py @@ -19,7 +19,6 @@ import pathlib from typing import Any, Mapping, Optional -# import kfp.deprecated as kfp from kfp.deprecated._config import COMPILING_FOR_V2 from kfp.deprecated.components import _structures, _data_passing from kfp.deprecated.components import _components diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py index 8b135ce8640..0ee0c85f946 100644 --- a/sdk/python/tests/dsl/component_tests.py +++ b/sdk/python/tests/dsl/component_tests.py @@ -23,729 +23,7 @@ from kfp.deprecated.dsl import ContainerOp, Pipeline, PipelineParam from kfp.deprecated.components.structures import ComponentSpec, InputSpec, OutputSpec - -class TestPythonComponent(unittest.TestCase): - - def test_component_metadata(self): - """Test component decorator metadata.""" - - class MockContainerOp: - - def _set_metadata(self, component_meta): - self._metadata = component_meta - - @component - def componentA( - a: {'ArtifactA': { - 'file_type': 'csv' - }}, - b: Integer() = 12, - c: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} = 'gs://hello/world' - ) -> { - 'model': Integer() - }: - return MockContainerOp() - - containerOp = componentA(1, 2, c=3) - - golden_meta = ComponentSpec(name='ComponentA', inputs=[], outputs=[]) - golden_meta.inputs.append( - InputSpec(name='a', type={'ArtifactA': { - 'file_type': 'csv' - }})) - golden_meta.inputs.append( - InputSpec( - name='b', - type={ - 'Integer': { - 'openapi_schema_validator': { - "type": "integer" - } - } - }, - default="12", - optional=True)) - golden_meta.inputs.append( - InputSpec( - name='c', - type={'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }}, - default='gs://hello/world', - optional=True)) - golden_meta.outputs.append( - OutputSpec( - name='model', - type={ - 'Integer': { - 'openapi_schema_validator': { - "type": "integer" - } - } - })) - - self.assertEqual(containerOp._metadata, golden_meta) - - def test_component_metadata_standard_type_annotation(self): - """Test component decorator metadata.""" - - class MockContainerOp: - - def _set_metadata(self, component_meta): - self._metadata = component_meta - - @component - def componentA(a: float, b: List[int], c: Optional[str] = None) -> None: - return MockContainerOp() - - containerOp = componentA('str_value', '[1,2,3]') - - golden_meta = ComponentSpec(name='ComponentA', inputs=[], outputs=None) - golden_meta.inputs.append(InputSpec(name='a', type='Float')) - golden_meta.inputs.append( - InputSpec( - name='b', - type='typing.List[int]' if sys.version_info >= - (3, 7) else 'List')) - golden_meta.inputs.append( - InputSpec(name='c', type='String', default=None, optional=True)) - - self.assertEqual(containerOp._metadata, golden_meta) - - def test_python_component_decorator(self): - # Deprecated - from kfp.deprecated.dsl import python_component - from kfp.deprecated.components import create_component_from_func - - expected_name = 'Sum component name' - expected_description = 'Sum component description' - expected_image = 'org/image' - - @python_component( - name=expected_name, - description=expected_description, - base_image=expected_image) - def add_two_numbers_decorated( - a: float, - b: float, - ) -> float: - """Returns sum of two arguments.""" - return a + b - - op = create_component_from_func(add_two_numbers_decorated) - - component_spec = op.component_spec - self.assertEqual(component_spec.name, expected_name) - self.assertEqual(component_spec.description.strip(), - expected_description.strip()) - self.assertEqual(component_spec.implementation.container.image, - expected_image) - - def test_type_check_with_same_representation(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': GCSPath(), - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'GcsUri' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-a', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - field_y: 'GcsUri', # noqa: F821 TODO - field_z: GCSPath() - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-b', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_different_represenation(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': 'GCSPath', - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, field_y: Integer(), field_z: GCSPath() - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_inconsistent_types_property_value(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'csv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - with self.assertRaises(InconsistentTypeException): - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_inconsistent_types_type_name(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type_a': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - with self.assertRaises(InconsistentTypeException): - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_without_types(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - } - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_nonnamed_inputs(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - } - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - a.outputs['field_n'], - field_z=a.outputs['field_m'], - field_y=a.outputs['field_o']) - - def test_type_check_with_inconsistent_types_disabled(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = False - - @component - def a_op(field_l: Integer()) -> { - 'field_m': { - 'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - } - }, - 'field_n': { - 'customized_type': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type_a': { - 'property_a': 'value_a', - 'property_b': 'value_b' - } - }, field_y: Integer(), - field_z: {'ArtifactB': { - 'path_type': 'file', - 'file_type': 'tsv' - }} - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_openapi_schema(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': 'GCSPath', - 'field_n': { - 'customized_type': { - 'openapi_schema_validator': - '{"type": "string", "pattern": "^gs://.*$"}' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'openapi_schema_validator': - '{"type": "string", "pattern": "^gs://.*$"}' - } - }, field_y: Integer(), field_z: GCSPath() - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - def test_type_check_with_ignore_type(self): - """Test type check at the decorator.""" - kfp.TYPE_CHECK = True - - @component - def a_op(field_l: Integer()) -> { - 'field_m': 'GCSPath', - 'field_n': { - 'customized_type': { - 'openapi_schema_validator': - '{"type": "string", "pattern": "^gs://.*$"}' - } - }, - 'field_o': 'Integer' - }: - return ContainerOp( - name='operator a', - image='gcr.io/ml-pipeline/component-b', - arguments=[ - '--field-l', - field_l, - ], - file_outputs={ - 'field_m': '/schema.txt', - 'field_n': '/feature.txt', - 'field_o': '/output.txt' - }) - - @component - def b_op( - field_x: { - 'customized_type': { - 'openapi_schema_validator': - '{"type": "string", "pattern": "^gcs://.*$"}' - } - }, field_y: Integer(), field_z: GCSPath() - ) -> { - 'output_model_uri': 'GcsUri' - }: - return ContainerOp( - name='operator b', - image='gcr.io/ml-pipeline/component-a', - command=[ - 'python3', - field_x, - ], - arguments=[ - '--field-y', - field_y, - '--field-z', - field_z, - ], - file_outputs={ - 'output_model_uri': '/schema.txt', - }) - - a = a_op(field_l=12) - with self.assertRaises(InconsistentTypeException): - b = b_op( - field_x=a.outputs['field_n'], - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - b = b_op( - field_x=a.outputs['field_n'].ignore_type(), - field_y=a.outputs['field_o'], - field_z=a.outputs['field_m']) - - +@unittest.skip("deprecated") class TestGraphComponent(unittest.TestCase): def test_graphcomponent_basic(self): From 11f014de7809aa646ef0e2e1dde43dfbd202c52e Mon Sep 17 00:00:00 2001 From: Yaqi Date: Mon, 14 Feb 2022 11:43:58 -0800 Subject: [PATCH 14/39] fix samples test --- samples/test/after_test.py | 2 +- samples/test/cache_v2_compatible_test.py | 1 - samples/test/fail.py | 2 +- samples/test/fail_parameter_value_missing.py | 2 +- samples/test/fail_parameter_value_missing_test.py | 2 +- samples/test/fail_test.py | 2 +- samples/test/legacy_data_passing.py | 4 ++-- samples/test/legacy_data_passing_test.py | 2 +- samples/test/legacy_exit_handler.py | 6 +++--- samples/test/legacy_exit_handler_test.py | 2 +- .../test/lightweight_python_functions_v2_pipeline_test.py | 2 +- .../lightweight_python_functions_v2_with_outputs_test.py | 2 +- samples/test/metrics_visualization_v1.py | 2 +- samples/test/metrics_visualization_v1_test.py | 2 +- samples/test/metrics_visualization_v2_test.py | 2 +- samples/test/parameter_with_format.py | 4 ++-- samples/test/parameter_with_format_test.py | 2 +- samples/test/placeholder_concat_test.py | 2 +- samples/test/placeholder_if.py | 4 ++-- samples/test/placeholder_if_test.py | 2 +- samples/test/reused_component.py | 4 ++-- samples/test/reused_component_test.py | 2 +- samples/test/two_step.py | 4 ++-- samples/test/two_step_test.py | 2 +- samples/test/two_step_with_uri_placeholder.py | 1 - samples/test/two_step_with_uri_placeholder_test.py | 2 +- samples/test/utils/kfp/samples/test/utils.py | 6 +++--- samples/v2/cache_test.py | 2 +- samples/v2/hello_world_test.py | 2 +- samples/v2/pipeline_with_importer_test.py | 2 +- samples/v2/producer_consumer_param_test.py | 2 +- 31 files changed, 38 insertions(+), 40 deletions(-) diff --git a/samples/test/after_test.py b/samples/test/after_test.py index 7c1c9fed563..04fdde001f3 100644 --- a/samples/test/after_test.py +++ b/samples/test/after_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .after import my_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/test/cache_v2_compatible_test.py b/samples/test/cache_v2_compatible_test.py index 9583fd0ccfa..aac32d627a8 100644 --- a/samples/test/cache_v2_compatible_test.py +++ b/samples/test/cache_v2_compatible_test.py @@ -22,7 +22,6 @@ import unittest import functools -import kfp import kfp_server_api from .two_step import two_step_pipeline diff --git a/samples/test/fail.py b/samples/test/fail.py index bd2b7c6800f..6533b2d8289 100644 --- a/samples/test/fail.py +++ b/samples/test/fail.py @@ -13,7 +13,7 @@ # limitations under the License. """Fail pipeline.""" -from kfp import components, dsl +from kfp.deprecated import components, dsl def fail(): diff --git a/samples/test/fail_parameter_value_missing.py b/samples/test/fail_parameter_value_missing.py index 9c408c6543f..6e905837e45 100644 --- a/samples/test/fail_parameter_value_missing.py +++ b/samples/test/fail_parameter_value_missing.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from kfp import dsl, components +from kfp.deprecated import dsl, components echo = components.load_component_from_text( """ diff --git a/samples/test/fail_parameter_value_missing_test.py b/samples/test/fail_parameter_value_missing_test.py index 1f1faf548e2..99c427e399a 100644 --- a/samples/test/fail_parameter_value_missing_test.py +++ b/samples/test/fail_parameter_value_missing_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .fail_parameter_value_missing import pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/test/fail_test.py b/samples/test/fail_test.py index a43a9db3b3b..ee7a489cf30 100644 --- a/samples/test/fail_test.py +++ b/samples/test/fail_test.py @@ -15,7 +15,7 @@ from __future__ import annotations import unittest -import kfp +import kfp.deprecated as kfp import kfp_server_api from ml_metadata.proto import Execution from .fail import fail_pipeline diff --git a/samples/test/legacy_data_passing.py b/samples/test/legacy_data_passing.py index bf3c9501930..e462f9b3b24 100644 --- a/samples/test/legacy_data_passing.py +++ b/samples/test/legacy_data_passing.py @@ -24,8 +24,8 @@ # The only restriction on types is that when both upstream output and downstream input have types, the types must match. # %% -import kfp -from kfp.components import create_component_from_func, InputPath, OutputPath +import kfp.deprecated as kfp +from kfp.deprecated.components import create_component_from_func, InputPath, OutputPath # Components diff --git a/samples/test/legacy_data_passing_test.py b/samples/test/legacy_data_passing_test.py index 1043054fba1..3952d024ebf 100644 --- a/samples/test/legacy_data_passing_test.py +++ b/samples/test/legacy_data_passing_test.py @@ -1,6 +1,6 @@ from .legacy_data_passing import data_passing_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase -from kfp.dsl import PipelineExecutionMode +from kfp.deprecated.dsl import PipelineExecutionMode run_pipeline_func([ TestCase( diff --git a/samples/test/legacy_exit_handler.py b/samples/test/legacy_exit_handler.py index 898a51d718a..df54f004855 100755 --- a/samples/test/legacy_exit_handler.py +++ b/samples/test/legacy_exit_handler.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp import dsl +import kfp.deprecated as kfp +from kfp.deprecated import dsl, compiler def gcs_download_op(url): @@ -54,4 +54,4 @@ def download_and_print(url='gs://ml-pipeline/shakespeare1.txt'): if __name__ == '__main__': - kfp.compiler.Compiler().compile(download_and_print, __file__ + '.yaml') + compiler.Compiler().compile(download_and_print, __file__ + '.yaml') diff --git a/samples/test/legacy_exit_handler_test.py b/samples/test/legacy_exit_handler_test.py index ae51ec7040f..51b22cf4c9e 100755 --- a/samples/test/legacy_exit_handler_test.py +++ b/samples/test/legacy_exit_handler_test.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .legacy_exit_handler import download_and_print from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/test/lightweight_python_functions_v2_pipeline_test.py b/samples/test/lightweight_python_functions_v2_pipeline_test.py index 62cede917aa..15d458b64b3 100644 --- a/samples/test/lightweight_python_functions_v2_pipeline_test.py +++ b/samples/test/lightweight_python_functions_v2_pipeline_test.py @@ -15,7 +15,7 @@ import unittest from pprint import pprint import kfp_server_api -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl from .lightweight_python_functions_v2_pipeline import pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient diff --git a/samples/test/lightweight_python_functions_v2_with_outputs_test.py b/samples/test/lightweight_python_functions_v2_with_outputs_test.py index 8a606e828d3..7d60c92b78c 100644 --- a/samples/test/lightweight_python_functions_v2_with_outputs_test.py +++ b/samples/test/lightweight_python_functions_v2_with_outputs_test.py @@ -14,7 +14,7 @@ from pprint import pprint import unittest -import kfp +import kfp.deprecated as kfp import kfp_server_api import os from minio import Minio diff --git a/samples/test/metrics_visualization_v1.py b/samples/test/metrics_visualization_v1.py index 0ce675f2917..5170194edd5 100644 --- a/samples/test/metrics_visualization_v1.py +++ b/samples/test/metrics_visualization_v1.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl +import kfp.deprecated.dsl as dsl from ..core.visualization.confusion_matrix import confusion_visualization from ..core.visualization.html import html_visualization from ..core.visualization.markdown import markdown_visualization diff --git a/samples/test/metrics_visualization_v1_test.py b/samples/test/metrics_visualization_v1_test.py index d371b96f3dc..1fa1809f0c2 100644 --- a/samples/test/metrics_visualization_v1_test.py +++ b/samples/test/metrics_visualization_v1_test.py @@ -16,7 +16,7 @@ from .metrics_visualization_v1 import metrics_visualization_v1_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase -import kfp +import kfp.deprecated as kfp run_pipeline_func([ TestCase( diff --git a/samples/test/metrics_visualization_v2_test.py b/samples/test/metrics_visualization_v2_test.py index ad1a8156dde..e30fc85b5a8 100644 --- a/samples/test/metrics_visualization_v2_test.py +++ b/samples/test/metrics_visualization_v2_test.py @@ -15,7 +15,7 @@ from __future__ import annotations import unittest import unittest.mock as mock -import kfp +import kfp.deprecated as kfp import kfp_server_api from .metrics_visualization_v2 import metrics_visualization_pipeline diff --git a/samples/test/parameter_with_format.py b/samples/test/parameter_with_format.py index 54b2a8fd6e0..6598bdcabe8 100644 --- a/samples/test/parameter_with_format.py +++ b/samples/test/parameter_with_format.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp import components -from kfp import dsl +from kfp.deprecated import components +from kfp.deprecated import dsl @components.create_component_from_func diff --git a/samples/test/parameter_with_format_test.py b/samples/test/parameter_with_format_test.py index c9ada5218b1..df99f1504e0 100644 --- a/samples/test/parameter_with_format_test.py +++ b/samples/test/parameter_with_format_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .parameter_with_format import my_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/test/placeholder_concat_test.py b/samples/test/placeholder_concat_test.py index 323ab3c14a7..fe741e76c14 100644 --- a/samples/test/placeholder_concat_test.py +++ b/samples/test/placeholder_concat_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .placeholder_concat import pipeline_with_concat_placeholder from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/test/placeholder_if.py b/samples/test/placeholder_if.py index bf0697a30f7..e0b5e04726f 100644 --- a/samples/test/placeholder_if.py +++ b/samples/test/placeholder_if.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp import components -from kfp import dsl +from kfp.deprecated import components +from kfp.deprecated import dsl component_op = components.load_component_from_text(''' name: Component with optional inputs diff --git a/samples/test/placeholder_if_test.py b/samples/test/placeholder_if_test.py index 2459133d8db..9669c7c868c 100644 --- a/samples/test/placeholder_if_test.py +++ b/samples/test/placeholder_if_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .placeholder_if import pipeline_both, pipeline_none # from .placeholder_if_v2 import pipeline_both as pipeline_both_v2, pipeline_none as pipeline_none_v2 from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/test/reused_component.py b/samples/test/reused_component.py index a4626ed4230..d2e710bc2ba 100644 --- a/samples/test/reused_component.py +++ b/samples/test/reused_component.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp import components -from kfp import dsl +from kfp.deprecated import components +from kfp.deprecated import dsl add_op = components.load_component_from_text( ''' diff --git a/samples/test/reused_component_test.py b/samples/test/reused_component_test.py index 0348444cb65..6864be8879e 100644 --- a/samples/test/reused_component_test.py +++ b/samples/test/reused_component_test.py @@ -14,7 +14,7 @@ import unittest -import kfp +import kfp.deprecated as kfp from .reused_component import my_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient diff --git a/samples/test/two_step.py b/samples/test/two_step.py index 170cf178ce1..00435fed51f 100644 --- a/samples/test/two_step.py +++ b/samples/test/two_step.py @@ -13,8 +13,8 @@ # limitations under the License. """Two step v2-compatible pipeline.""" -from kfp import components, dsl -from kfp.components import InputPath, OutputPath +from kfp.deprecated import components, dsl +from kfp.deprecated.components import InputPath, OutputPath def preprocess( diff --git a/samples/test/two_step_test.py b/samples/test/two_step_test.py index 9261fdd3fb1..d8187e89aa2 100644 --- a/samples/test/two_step_test.py +++ b/samples/test/two_step_test.py @@ -19,7 +19,7 @@ import unittest from pprint import pprint -import kfp +import kfp.deprecated as kfp import kfp_server_api from .two_step import two_step_pipeline diff --git a/samples/test/two_step_with_uri_placeholder.py b/samples/test/two_step_with_uri_placeholder.py index 4cc60d1410e..36a161991fc 100644 --- a/samples/test/two_step_with_uri_placeholder.py +++ b/samples/test/two_step_with_uri_placeholder.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """Two step v2-compatible pipeline with URI placeholders.""" -import kfp from kfp.v2 import components, dsl write_to_gcs_op = components.load_component_from_text(""" diff --git a/samples/test/two_step_with_uri_placeholder_test.py b/samples/test/two_step_with_uri_placeholder_test.py index 1eaa60272c4..a56b270d18d 100644 --- a/samples/test/two_step_with_uri_placeholder_test.py +++ b/samples/test/two_step_with_uri_placeholder_test.py @@ -17,7 +17,7 @@ from pprint import pprint from typing import Dict -import kfp +import kfp.deprecated as kfp import kfp_server_api from .two_step_with_uri_placeholder import two_step_with_uri_placeholder diff --git a/samples/test/utils/kfp/samples/test/utils.py b/samples/test/utils/kfp/samples/test/utils.py index 87ebba70f5f..021e24c798c 100644 --- a/samples/test/utils/kfp/samples/test/utils.py +++ b/samples/test/utils/kfp/samples/test/utils.py @@ -30,8 +30,8 @@ import nbformat from nbconvert import PythonExporter -import kfp -from kfp.onprem import add_default_resource_spec +import kfp.deprecated as kfp +from kfp.deprecated.onprem import add_default_resource_spec import kfp.v2.compiler import kfp_server_api from ml_metadata import metadata_store @@ -329,7 +329,7 @@ def _create_run(): else: package_path = tempfile.mktemp( suffix='.yaml', prefix="kfp_package") - from kfp.compiler.main import compile_pyfile + from kfp.deprecated.compiler.main import compile_pyfile compile_pyfile( pyfile=pyfile, output_path=package_path, diff --git a/samples/v2/cache_test.py b/samples/v2/cache_test.py index 785e1c7bd8d..a1fb533b121 100644 --- a/samples/v2/cache_test.py +++ b/samples/v2/cache_test.py @@ -22,7 +22,7 @@ import unittest import functools -import kfp +import kfp.deprecated as kfp import kfp_server_api from ..test.two_step import two_step_pipeline diff --git a/samples/v2/hello_world_test.py b/samples/v2/hello_world_test.py index bd02f0e9a28..dd6a2b548ee 100644 --- a/samples/v2/hello_world_test.py +++ b/samples/v2/hello_world_test.py @@ -17,7 +17,7 @@ import unittest -import kfp +import kfp.deprecated as kfp import kfp_server_api from ml_metadata.proto import Execution diff --git a/samples/v2/pipeline_with_importer_test.py b/samples/v2/pipeline_with_importer_test.py index da70046032a..dbab6e3b045 100644 --- a/samples/v2/pipeline_with_importer_test.py +++ b/samples/v2/pipeline_with_importer_test.py @@ -18,7 +18,7 @@ import unittest from pprint import pprint -import kfp +import kfp.deprecated as kfp import kfp_server_api from ml_metadata.proto import Execution diff --git a/samples/v2/producer_consumer_param_test.py b/samples/v2/producer_consumer_param_test.py index 36f8b6bf429..086e92dd22d 100644 --- a/samples/v2/producer_consumer_param_test.py +++ b/samples/v2/producer_consumer_param_test.py @@ -17,7 +17,7 @@ import unittest from pprint import pprint -import kfp +import kfp.deprecated as kfp import kfp_server_api from .producer_consumer_param import producer_consumer_param_pipeline From bd6e258f2ada141eb0b3ed2952b2572a7151f1f0 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Tue, 15 Feb 2022 00:50:39 -0800 Subject: [PATCH 15/39] sample test --- samples/core/caching/caching.ipynb | 14 +++++++------- samples/core/caching/caching_sample.py | 4 ++-- samples/core/caching/caching_test.py | 2 +- samples/core/condition/condition.py | 9 +++++---- samples/core/condition/condition_test.py | 2 +- samples/core/condition/nested_condition.py | 4 ++-- samples/core/condition/nested_condition_test.py | 2 +- .../continue_training_from_prod.py | 12 ++++++------ samples/core/dataflow/dataflow_test.py | 2 +- samples/core/dns_config/dns_config.py | 6 +++--- .../dsl_static_type_checking_test.py | 2 +- samples/core/execution_order/execution_order.py | 7 +++---- .../core/execution_order/execution_order_test.py | 2 +- samples/core/exit_handler/exit_handler.py | 9 ++++----- samples/core/exit_handler/exit_handler_test.py | 2 +- samples/core/helloworld/hello_world.py | 8 ++++---- samples/core/imagepullsecrets/imagepullsecrets.py | 7 ++++--- .../core/imagepullsecrets/imagepullsecrets_test.py | 2 +- .../lightweight_component_test.py | 2 +- .../core/output_a_directory/output_a_directory.py | 4 ++-- .../output_a_directory/output_a_directory_test.py | 2 +- samples/core/parallel_join/parallel_join.py | 6 +++--- .../parallelism_sub_dag/parallelism_sub_dag.py | 4 ++-- .../parallelism_sub_dag_test.py | 2 +- .../parallelism_sub_dag_with_op_output.py | 4 ++-- 25 files changed, 60 insertions(+), 60 deletions(-) diff --git a/samples/core/caching/caching.ipynb b/samples/core/caching/caching.ipynb index 5a077f92624..0a03f8e7614 100644 --- a/samples/core/caching/caching.ipynb +++ b/samples/core/caching/caching.ipynb @@ -1,14 +1,18 @@ { "cells": [ { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "kfp_endpoint = None\n", "\n", "import datetime\n", "import time\n", "\n", - "import kfp\n", - "from kfp.components import create_component_from_func\n", + "import kfp.deprecated as kfp\n", + "from kfp.deprecated.components import create_component_from_func\n", "\n", "\n", "@create_component_from_func\n", @@ -26,11 +30,7 @@ "def caching_pipeline(seconds: float = 60):\n", " # All outputs of successful executions are cached\n", " work_task = do_work_op(seconds)\n" - ], - "cell_type": "code", - "metadata": {}, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", diff --git a/samples/core/caching/caching_sample.py b/samples/core/caching/caching_sample.py index 28543dc3058..bd1220bddd9 100644 --- a/samples/core/caching/caching_sample.py +++ b/samples/core/caching/caching_sample.py @@ -3,8 +3,8 @@ import datetime import time -import kfp -from kfp.components import create_component_from_func +import kfp.deprecated as kfp +from kfp.deprecated.components import create_component_from_func @create_component_from_func diff --git a/samples/core/caching/caching_test.py b/samples/core/caching/caching_test.py index 986b004f86a..43c99709149 100644 --- a/samples/core/caching/caching_test.py +++ b/samples/core/caching/caching_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/condition/condition.py b/samples/core/condition/condition.py index 6434a02b3cd..376a2bd5378 100644 --- a/samples/core/condition/condition.py +++ b/samples/core/condition/condition.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp import components -from kfp import dsl +import kfp.deprecated as kfp +from kfp.deprecated import components +from kfp.deprecated import dsl +from kfp.deprecated import compiler def flip_coin(force_flip_result: str = '') -> str: @@ -48,4 +49,4 @@ def condition(text: str = 'condition test', force_flip_result: str = ''): if __name__ == '__main__': - kfp.compiler.Compiler().compile(condition, __file__ + '.yaml') + compiler.Compiler().compile(condition, __file__ + '.yaml') diff --git a/samples/core/condition/condition_test.py b/samples/core/condition/condition_test.py index adba9888c9b..a4551e047a2 100644 --- a/samples/core/condition/condition_test.py +++ b/samples/core/condition/condition_test.py @@ -15,7 +15,7 @@ from __future__ import annotations import unittest -import kfp +import kfp.deprecated as kfp import kfp_server_api from ml_metadata.proto import Execution from .condition import condition diff --git a/samples/core/condition/nested_condition.py b/samples/core/condition/nested_condition.py index 4f33a793edd..cd10e143c5e 100644 --- a/samples/core/condition/nested_condition.py +++ b/samples/core/condition/nested_condition.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp import components -from kfp import dsl +from kfp.deprecated import components +from kfp.deprecated import dsl @components.create_component_from_func diff --git a/samples/core/condition/nested_condition_test.py b/samples/core/condition/nested_condition_test.py index b8c50b502b4..7699d957425 100644 --- a/samples/core/condition/nested_condition_test.py +++ b/samples/core/condition/nested_condition_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .nested_condition import my_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/core/continue_training_from_prod/continue_training_from_prod.py b/samples/core/continue_training_from_prod/continue_training_from_prod.py index ab39b3ab08c..10be8035c08 100644 --- a/samples/core/continue_training_from_prod/continue_training_from_prod.py +++ b/samples/core/continue_training_from_prod/continue_training_from_prod.py @@ -10,8 +10,8 @@ kfp_endpoint=None -import kfp -from kfp import components +import kfp.deprecated as kfp +from kfp.deprecated import components chicago_taxi_dataset_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml') @@ -52,7 +52,7 @@ def continuous_training_pipeline( table=testing_data, transform_code='''df = df[["tips"]]''', ).set_display_name('True values').output - + true_values = drop_header_op(true_values_table).output # Getting the active prod model @@ -82,7 +82,7 @@ def continuous_training_pipeline( label_column=0, ).output - # Calculating the regression metrics + # Calculating the regression metrics metrics_task = calculate_regression_metrics_from_csv_op( true_values=true_values, predicted_values=predictions, @@ -106,7 +106,7 @@ def continuous_training_pipeline( with kfp.dsl.Condition(prod_model_uri != ""): # Downloading the model prod_model = download_from_gcs_op(prod_model_uri).output - + # Training model = xgboost_train_on_csv_op( training_data=training_data, @@ -123,7 +123,7 @@ def continuous_training_pipeline( label_column=0, ).output - # Calculating the regression metrics + # Calculating the regression metrics metrics_task = calculate_regression_metrics_from_csv_op( true_values=true_values, predicted_values=predictions, diff --git a/samples/core/dataflow/dataflow_test.py b/samples/core/dataflow/dataflow_test.py index fea1e837ad3..3f02123e482 100644 --- a/samples/core/dataflow/dataflow_test.py +++ b/samples/core/dataflow/dataflow_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/dns_config/dns_config.py b/samples/core/dns_config/dns_config.py index ff9f8040954..7621faf0298 100644 --- a/samples/core/dns_config/dns_config.py +++ b/samples/core/dns_config/dns_config.py @@ -14,8 +14,8 @@ # limitations under the License. -import kfp -from kfp import dsl +import kfp.deprecated as kfp +from kfp.deprecated import dsl, compiler from kubernetes.client.models import V1PodDNSConfig, V1PodDNSConfigOption @@ -43,7 +43,7 @@ def dns_config_pipeline(): options=[V1PodDNSConfigOption(name="ndots", value="2")] )) - kfp.compiler.Compiler().compile( + compiler.Compiler().compile( dns_config_pipeline, __file__ + '.yaml', pipeline_conf=pipeline_conf diff --git a/samples/core/dsl_static_type_checking/dsl_static_type_checking_test.py b/samples/core/dsl_static_type_checking/dsl_static_type_checking_test.py index 2278dfa42fc..c5e02bcb7b7 100644 --- a/samples/core/dsl_static_type_checking/dsl_static_type_checking_test.py +++ b/samples/core/dsl_static_type_checking/dsl_static_type_checking_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/execution_order/execution_order.py b/samples/core/execution_order/execution_order.py index 39f61955325..229364ccbac 100755 --- a/samples/core/execution_order/execution_order.py +++ b/samples/core/execution_order/execution_order.py @@ -14,9 +14,8 @@ # limitations under the License. -import kfp -from kfp import dsl -import kfp.components as comp +from kfp.deprecated import dsl, compiler +import kfp.deprecated.components as comp @comp.create_component_from_func @@ -40,4 +39,4 @@ def execution_order_pipeline(text1: str='message 1', text2: str='message 2'): step2_task.after(step1_task) if __name__ == '__main__': - kfp.compiler.Compiler().compile(execution_order_pipeline, __file__ + '.yaml') \ No newline at end of file + compiler.Compiler().compile(execution_order_pipeline, __file__ + '.yaml') \ No newline at end of file diff --git a/samples/core/execution_order/execution_order_test.py b/samples/core/execution_order/execution_order_test.py index e48fa49b913..64da9040c97 100644 --- a/samples/core/execution_order/execution_order_test.py +++ b/samples/core/execution_order/execution_order_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/exit_handler/exit_handler.py b/samples/core/exit_handler/exit_handler.py index c342e3c6fe8..2de097bba1c 100755 --- a/samples/core/exit_handler/exit_handler.py +++ b/samples/core/exit_handler/exit_handler.py @@ -13,11 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp import dsl, components -from kfp.components import InputPath +from kfp.deprecated import dsl, components, compiler +from kfp.deprecated.components import InputPath, load_component_from_url -gcs_download_op = kfp.components.load_component_from_url( +gcs_download_op = load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/961b17fa6844e1d79e5d3686bb557d830d7b5a95/components/google-cloud/storage/download_blob/component.yaml' ) @@ -50,4 +49,4 @@ def pipeline_exit_handler(url: str = 'gs://ml-pipeline/shakespeare1.txt'): if __name__ == '__main__': - kfp.compiler.Compiler().compile(pipeline_exit_handler, __file__ + '.yaml') + compiler.Compiler().compile(pipeline_exit_handler, __file__ + '.yaml') diff --git a/samples/core/exit_handler/exit_handler_test.py b/samples/core/exit_handler/exit_handler_test.py index 723d71d90da..8f0b8d8856f 100644 --- a/samples/core/exit_handler/exit_handler_test.py +++ b/samples/core/exit_handler/exit_handler_test.py @@ -17,7 +17,7 @@ import unittest from pprint import pprint -import kfp +import kfp.deprecated as kfp import kfp_server_api from .exit_handler import pipeline_exit_handler diff --git a/samples/core/helloworld/hello_world.py b/samples/core/helloworld/hello_world.py index 4b06dbaa17e..605684b08b8 100644 --- a/samples/core/helloworld/hello_world.py +++ b/samples/core/helloworld/hello_world.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp import dsl -import kfp.components as comp +import kfp.deprecated as kfp +from kfp.deprecated import dsl, compiler +import kfp.deprecated.components as comp @comp.create_component_from_func @@ -29,4 +29,4 @@ def hello_world_pipeline(): echo_task = echo_op() if __name__ == '__main__': - kfp.compiler.Compiler().compile(hello_world_pipeline, __file__ + '.yaml') \ No newline at end of file + compiler.Compiler().compile(hello_world_pipeline, __file__ + '.yaml') \ No newline at end of file diff --git a/samples/core/imagepullsecrets/imagepullsecrets.py b/samples/core/imagepullsecrets/imagepullsecrets.py index 89ebb4ec2be..2fffc4a5816 100644 --- a/samples/core/imagepullsecrets/imagepullsecrets.py +++ b/samples/core/imagepullsecrets/imagepullsecrets.py @@ -15,8 +15,9 @@ container registry. """ -import kfp -import kfp.dsl as dsl +import kfp.deprecated as kfp +import kfp.deprecated.dsl as dsl +from kfp.deprecated import compiler from kubernetes import client as k8s_client @@ -55,4 +56,4 @@ def save_most_frequent_word(message: str): .set_image_pull_secrets([k8s_client.V1ObjectReference(name="secretA")]) if __name__ == '__main__': - kfp.compiler.Compiler().compile(save_most_frequent_word, __file__ + '.yaml') + compiler.Compiler().compile(save_most_frequent_word, __file__ + '.yaml') diff --git a/samples/core/imagepullsecrets/imagepullsecrets_test.py b/samples/core/imagepullsecrets/imagepullsecrets_test.py index 71836804b2e..7bf33720027 100644 --- a/samples/core/imagepullsecrets/imagepullsecrets_test.py +++ b/samples/core/imagepullsecrets/imagepullsecrets_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/lightweight_component/lightweight_component_test.py b/samples/core/lightweight_component/lightweight_component_test.py index 3bef00735be..c4bb16ae097 100644 --- a/samples/core/lightweight_component/lightweight_component_test.py +++ b/samples/core/lightweight_component/lightweight_component_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/output_a_directory/output_a_directory.py b/samples/core/output_a_directory/output_a_directory.py index 2b0d1257606..3db9d3cf358 100644 --- a/samples/core/output_a_directory/output_a_directory.py +++ b/samples/core/output_a_directory/output_a_directory.py @@ -19,8 +19,8 @@ # To output a directory, create a new directory at the output path location. import os -import kfp -from kfp.components import create_component_from_func, load_component_from_text, InputPath, OutputPath +import kfp.deprecated as kfp +from kfp.deprecated.components import create_component_from_func, load_component_from_text, InputPath, OutputPath import kfp.v2 as v2 from kfp.v2.dsl import Input, Output, Artifact diff --git a/samples/core/output_a_directory/output_a_directory_test.py b/samples/core/output_a_directory/output_a_directory_test.py index 518079f2118..1ecbda7dedc 100644 --- a/samples/core/output_a_directory/output_a_directory_test.py +++ b/samples/core/output_a_directory/output_a_directory_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .output_a_directory import dir_pipeline, dir_pipeline_v2 from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/core/parallel_join/parallel_join.py b/samples/core/parallel_join/parallel_join.py index bad59ab1dcc..a5f7a3d3309 100755 --- a/samples/core/parallel_join/parallel_join.py +++ b/samples/core/parallel_join/parallel_join.py @@ -14,8 +14,8 @@ # limitations under the License. -import kfp -from kfp import dsl +import kfp.deprecated as kfp +from kfp.deprecated import dsl, compiler def gcs_download_op(url): return dsl.ContainerOp( @@ -54,4 +54,4 @@ def download_and_join( echo_task = echo2_op(download1_task.output, download2_task.output) if __name__ == '__main__': - kfp.compiler.Compiler().compile(download_and_join, __file__ + '.yaml') + compiler.Compiler().compile(download_and_join, __file__ + '.yaml') diff --git a/samples/core/parallelism_sub_dag/parallelism_sub_dag.py b/samples/core/parallelism_sub_dag/parallelism_sub_dag.py index a03130155eb..968e556f3dc 100644 --- a/samples/core/parallelism_sub_dag/parallelism_sub_dag.py +++ b/samples/core/parallelism_sub_dag/parallelism_sub_dag.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -import kfp +import kfp.deprecated.dsl as dsl +import kfp.deprecated as kfp @kfp.components.create_component_from_func def print_op(s: str): diff --git a/samples/core/parallelism_sub_dag/parallelism_sub_dag_test.py b/samples/core/parallelism_sub_dag/parallelism_sub_dag_test.py index a7667662ee0..c6d92e2ed5b 100644 --- a/samples/core/parallelism_sub_dag/parallelism_sub_dag_test.py +++ b/samples/core/parallelism_sub_dag/parallelism_sub_dag_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/parallelism_sub_dag/parallelism_sub_dag_with_op_output.py b/samples/core/parallelism_sub_dag/parallelism_sub_dag_with_op_output.py index f4ae56c4b31..158f80b62b7 100644 --- a/samples/core/parallelism_sub_dag/parallelism_sub_dag_with_op_output.py +++ b/samples/core/parallelism_sub_dag/parallelism_sub_dag_with_op_output.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -import kfp +import kfp.deprecated.dsl as dsl +import kfp.deprecated as kfp @kfp.components.create_component_from_func def print_op(s: str): From 322e8a92cc554a6577a7b178fd7deaa3cdd2379c Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 16 Feb 2022 00:25:34 -0800 Subject: [PATCH 16/39] fix samples --- ...- Retail product stockout prediction.ipynb | 6 +-- samples/core/XGBoost/xgboost_sample.py | 4 +- samples/core/XGBoost/xgboost_sample_test.py | 2 +- samples/core/condition/condition.py | 1 - samples/core/dataflow/dataflow.ipynb | 8 ++-- .../dsl_static_type_checking.ipynb | 42 +++++++++++-------- samples/core/helloworld/hello_world.py | 1 - .../core/imagepullsecrets/imagepullsecrets.py | 1 - .../kfp_env_validation.ipynb | 16 +++---- .../kubeflow_tf_serving.ipynb | 7 ++-- .../lightweight_component.ipynb | 12 ++++-- samples/core/loop_output/loop_output.py | 4 +- samples/core/loop_output/loop_output_test.py | 2 +- .../core/loop_parallelism/loop_parallelism.py | 8 ++-- .../loop_parallelism/loop_parallelism_test.py | 2 +- samples/core/loop_parameter/loop_parameter.py | 3 +- .../loop_parameter/loop_parameter_test.py | 2 +- samples/core/loop_static/loop_static.py | 3 +- samples/core/loop_static/loop_static_test.py | 2 +- .../multiple_outputs/multiple_outputs.ipynb | 6 +-- .../multiple_outputs/multiple_outputs_test.py | 2 +- samples/core/parallel_join/parallel_join.py | 1 - .../parameterized_tfx_oss.py | 6 +-- .../parameterized_tfx_oss_test.py | 2 +- .../pipeline_parallelism_limits.py | 8 ++-- .../pipeline_transformers.py | 7 ++-- .../pipeline_transformers_test.py | 2 +- .../preemptible_tpu_gpu.py | 4 +- .../preemptible_tpu_gpu_test.py | 2 +- samples/core/recursion/recursion.py | 7 ++-- samples/core/resource_ops/resource_ops.py | 7 ++-- .../core/resource_ops/resource_ops_test.py | 2 +- samples/core/resource_spec/resource_spec.py | 4 +- .../core/resource_spec/resource_spec_test.py | 2 +- .../resource_spec/runtime_resource_request.py | 7 ++-- .../runtime_resource_request_gpu.py | 5 +-- .../runtime_resource_request_test.py | 2 +- samples/core/retry/retry.py | 7 ++-- samples/core/retry/retry_test.py | 2 +- samples/core/secret/secret.py | 5 +-- samples/core/secret/secret_test.py | 2 +- samples/core/sequential/sequential.py | 5 +-- samples/core/sidecar/sidecar.py | 5 +-- samples/core/sidecar/sidecar_test.py | 2 +- .../core/train_until_good/train_until_good.py | 8 ++-- samples/core/use_run_info/use_run_id.py | 12 +++--- samples/core/use_run_info/use_run_id_test.py | 2 +- .../core/visualization/confusion_matrix.py | 10 ++--- .../visualization/confusion_matrix_test.py | 2 +- samples/core/visualization/html.py | 5 +-- samples/core/visualization/html_test.py | 2 +- samples/core/visualization/markdown.py | 4 +- samples/core/visualization/markdown_test.py | 2 +- samples/core/visualization/roc.py | 4 +- samples/core/visualization/roc_test.py | 2 +- samples/core/visualization/table.py | 6 +-- samples/core/visualization/table_test.py | 2 +- samples/core/visualization/tensorboard_gcs.py | 4 +- .../core/visualization/tensorboard_minio.py | 6 +-- .../visualization/tensorboard_minio_test.py | 2 +- samples/core/volume_ops/volume_ops.py | 8 ++-- .../volume_snapshot_ops.py | 5 +-- .../volume_snapshot_ops_test.py | 2 +- .../xgboost_training_cm.py | 13 +++--- 64 files changed, 156 insertions(+), 175 deletions(-) diff --git a/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb b/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb index 2eb589b77b6..d3ef3fc176f 100644 --- a/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb +++ b/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb @@ -29,7 +29,7 @@ "source": [ "# AutoML Tables components\n", "\n", - "from kfp.components import load_component_from_url\n", + "from kfp.deprecated.components import load_component_from_url\n", "\n", "automl_create_dataset_for_tables_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/create_dataset_for_tables/component.yaml')\n", "automl_import_data_from_bigquery_source_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/import_data_from_bigquery/component.yaml')\n", @@ -47,7 +47,7 @@ "outputs": [], "source": [ "# Define the pipeline\n", - "import kfp\n", + "import kfp.deprecated as kfp\n", "\n", "def retail_product_stockout_prediction_pipeline_gcs(\n", " gcp_project_id: str,\n", @@ -161,4 +161,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/samples/core/XGBoost/xgboost_sample.py b/samples/core/XGBoost/xgboost_sample.py index 9b7382f83d8..d44cfaa73ab 100644 --- a/samples/core/XGBoost/xgboost_sample.py +++ b/samples/core/XGBoost/xgboost_sample.py @@ -1,5 +1,5 @@ -import kfp -from kfp import components +import kfp.deprecated as kfp +from kfp.deprecated import components chicago_taxi_dataset_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml' diff --git a/samples/core/XGBoost/xgboost_sample_test.py b/samples/core/XGBoost/xgboost_sample_test.py index 03ba8252595..44e2c2a6b74 100644 --- a/samples/core/XGBoost/xgboost_sample_test.py +++ b/samples/core/XGBoost/xgboost_sample_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .xgboost_sample import xgboost_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/core/condition/condition.py b/samples/core/condition/condition.py index 376a2bd5378..0a5f861d470 100644 --- a/samples/core/condition/condition.py +++ b/samples/core/condition/condition.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.deprecated as kfp from kfp.deprecated import components from kfp.deprecated import dsl from kfp.deprecated import compiler diff --git a/samples/core/dataflow/dataflow.ipynb b/samples/core/dataflow/dataflow.ipynb index 238580301f3..39a0d7b1d78 100644 --- a/samples/core/dataflow/dataflow.ipynb +++ b/samples/core/dataflow/dataflow.ipynb @@ -125,7 +125,7 @@ }, "outputs": [], "source": [ - "import kfp.components as comp\n", + "import kfp.deprecated.components as comp\n", "\n", "dataflow_python_op = comp.load_component_from_url(\n", " 'https://raw.githubusercontent.com/kubeflow/pipelines/1.7.0-rc.3/components/gcp/dataflow/launch_python/component.yaml')" @@ -325,8 +325,8 @@ "metadata": {}, "outputs": [], "source": [ - "import kfp\n", - "import kfp.dsl as dsl\n", + "import kfp.deprecated as kfp\n", + "from kfp.deprecated import dsl, Client\n", "import json\n", "@dsl.pipeline(\n", " name='dataflow-launch-python-pipeline',\n", @@ -392,7 +392,7 @@ } ], "source": [ - "kfp.Client().create_run_from_pipeline_func(pipeline, arguments={})" + "Client().create_run_from_pipeline_func(pipeline, arguments={})" ] }, { diff --git a/samples/core/dsl_static_type_checking/dsl_static_type_checking.ipynb b/samples/core/dsl_static_type_checking/dsl_static_type_checking.ipynb index 543ff6e5d84..6c5aeb789f0 100644 --- a/samples/core/dsl_static_type_checking/dsl_static_type_checking.ipynb +++ b/samples/core/dsl_static_type_checking/dsl_static_type_checking.ipynb @@ -107,7 +107,11 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {"tags":["skip-in-test"]}, + "metadata": { + "tags": [ + "skip-in-test" + ] + }, "outputs": [ { "name": "stdout", @@ -245,9 +249,10 @@ "metadata": {}, "outputs": [], "source": [ - "import kfp.components as comp\n", - "import kfp.dsl as dsl\n", - "import kfp.compiler as compiler\n", + "from kfp.deprecated import components as comp\n", + "from kfp.deprecated import dsl\n", + "from kfp.deprecated import compiler\n", + "\n", "# The components are loaded as task factories that generate container_ops.\n", "task_factory_a = comp.load_component_from_text(text=component_a)\n", "task_factory_b = comp.load_component_from_text(text=component_b)\n", @@ -350,12 +355,13 @@ } ], "source": [ - "import kfp.components as comp\n", - "import kfp.dsl as dsl\n", - "import kfp.compiler as compiler\n", - "from kfp.dsl.types import InconsistentTypeException\n", - "task_factory_a = comp.load_component_from_text(text=component_a)\n", - "task_factory_b = comp.load_component_from_text(text=component_b)\n", + "from kfp.deprecated import components\n", + "from kfp.deprecated import dsl\n", + "from kfp.deprecated import compiler\n", + "from kfp.deprecated.dsl.types import InconsistentTypeException\n", + "\n", + "task_factory_a = components.load_component_from_text(text=component_a)\n", + "task_factory_b = components.load_component_from_text(text=component_b)\n", "\n", "#Use the component as part of the pipeline\n", "@dsl.pipeline(name='type-check-b',\n", @@ -407,9 +413,9 @@ "metadata": {}, "outputs": [], "source": [ - "from kfp.dsl import component\n", - "from kfp.dsl.types import Integer, GCSPath\n", - "from kfp.dsl import ContainerOp\n", + "from kfp.deprecated.dsl import component\n", + "from kfp.deprecated.dsl.types import Integer, GCSPath\n", + "from kfp.deprecated.dsl import ContainerOp\n", "# when components are defined based on the component decorator,\n", "# the type information is annotated to the input or function returns.\n", "# There are two ways to define the type: string or a dictionary with the openapi_schema_validator property\n", @@ -497,9 +503,9 @@ "metadata": {}, "outputs": [], "source": [ - "from kfp.dsl import component\n", - "from kfp.dsl.types import Integer, GCSPath\n", - "from kfp.dsl import ContainerOp\n", + "from kfp.deprecated.dsl import component\n", + "from kfp.deprecated.dsl.types import Integer, GCSPath\n", + "from kfp.deprecated.dsl import ContainerOp\n", "# task_factory_a outputs an input field_m with the openapi_schema_validator different\n", "# from the task_factory_b's input field_z.\n", "# One is gs:// and the other is gcs://\n", @@ -796,10 +802,10 @@ "pycharm": { "stem_cell": { "cell_type": "raw", - "source": [], "metadata": { "collapsed": false - } + }, + "source": [] } } }, diff --git a/samples/core/helloworld/hello_world.py b/samples/core/helloworld/hello_world.py index 605684b08b8..9ef87faf3fa 100644 --- a/samples/core/helloworld/hello_world.py +++ b/samples/core/helloworld/hello_world.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.deprecated as kfp from kfp.deprecated import dsl, compiler import kfp.deprecated.components as comp diff --git a/samples/core/imagepullsecrets/imagepullsecrets.py b/samples/core/imagepullsecrets/imagepullsecrets.py index 2fffc4a5816..9eb89bf0c88 100644 --- a/samples/core/imagepullsecrets/imagepullsecrets.py +++ b/samples/core/imagepullsecrets/imagepullsecrets.py @@ -15,7 +15,6 @@ container registry. """ -import kfp.deprecated as kfp import kfp.deprecated.dsl as dsl from kfp.deprecated import compiler from kubernetes import client as k8s_client diff --git a/samples/core/kfp_env_validation/kfp_env_validation.ipynb b/samples/core/kfp_env_validation/kfp_env_validation.ipynb index a85805bb704..ac0b8a51309 100644 --- a/samples/core/kfp_env_validation/kfp_env_validation.ipynb +++ b/samples/core/kfp_env_validation/kfp_env_validation.ipynb @@ -100,7 +100,7 @@ " import sys\n", " from typing import List, Text\n", " import os\n", - " from kfp.cli.diagnose_me import gcp\n", + " from kfp.deprecated.cli.diagnose_me import gcp\n", "\n", " # Get the project ID\n", " project_config = gcp.get_gcp_configuration(\n", @@ -155,7 +155,7 @@ " import sys\n", " from typing import List, Text \n", " import os\n", - " from kfp.cli.diagnose_me import gcp\n", + " from kfp.deprecated.cli.diagnose_me import gcp\n", " import json\n", " # Get the project ID\n", " project_config = gcp.get_gcp_configuration(gcp.Commands.GET_GCLOUD_DEFAULT,human_readable=False)\n", @@ -225,7 +225,7 @@ " import sys\n", " from typing import List, Text \n", " import os\n", - " from kfp.cli.diagnose_me import gcp\n", + " from kfp.deprecated.cli.diagnose_me import gcp\n", " \n", " # Get the project ID\n", " project_config = gcp.get_gcp_configuration(gcp.Commands.GET_GCLOUD_DEFAULT,human_readable=False)\n", @@ -271,7 +271,7 @@ "metadata": {}, "outputs": [], "source": [ - "import kfp.components as comp\n", + "import kfp.deprecated.components as comp\n", "\n", "run_diagnose_me_op = comp.func_to_container_op(\n", " run_diagnose_me, base_image='google/cloud-sdk:279.0.0')\n", @@ -293,8 +293,8 @@ "metadata": {}, "outputs": [], "source": [ - "from kfp.gcp import use_gcp_secret\n", - "from kfp import dsl\n", + "from kfp.deprecated.gcp import use_gcp_secret\n", + "from kfp.deprecated import dsl\n", "\n", "@dsl.pipeline(\n", " name='verify-kfp-env',\n", @@ -326,8 +326,8 @@ "metadata": {}, "outputs": [], "source": [ - "import kfp\n", - "client = kfp.Client(host='')" + "from kfp.deprecated import Client\n", + "client = Client(host='')" ] }, { diff --git a/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb b/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb index b7a2bbe87e4..75906cb0695 100644 --- a/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb +++ b/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb @@ -283,8 +283,7 @@ "metadata": {}, "outputs": [], "source": [ - "import kfp\n", - "import kfp.dsl as dsl\n", + "from kfp.deprecated import dsl, Client\n", "\n", "# The pipeline definition\n", "@dsl.pipeline(\n", @@ -321,7 +320,7 @@ } ], "source": [ - "kfp.Client().create_run_from_pipeline_func(model_server, arguments={})\n", + "Client().create_run_from_pipeline_func(model_server, arguments={})\n", "\n", "#vvvvvvvvv This link leads to the run information page. (Note: There is a bug in JupyterLab that modifies the URL and makes the link stop working)" ] @@ -348,4 +347,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/samples/core/lightweight_component/lightweight_component.ipynb b/samples/core/lightweight_component/lightweight_component.ipynb index 85ea7648cf5..ff50886b11f 100644 --- a/samples/core/lightweight_component/lightweight_component.ipynb +++ b/samples/core/lightweight_component/lightweight_component.ipynb @@ -39,8 +39,8 @@ "metadata": {}, "outputs": [], "source": [ - "import kfp\n", - "import kfp.components as components" + "import kfp.deprecated as kfp\n", + "import kfp.deprecated.components as components" ] }, { @@ -142,7 +142,11 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {"tags": ["skip-in-test"]}, + "metadata": { + "tags": [ + "skip-in-test" + ] + }, "outputs": [ { "data": { @@ -191,7 +195,7 @@ "metadata": {}, "outputs": [], "source": [ - "import kfp.dsl as dsl\n", + "import kfp.deprecated.dsl as dsl\n", "@dsl.pipeline(\n", " name='calculation-pipeline',\n", " description='A toy pipeline that performs arithmetic calculations.'\n", diff --git a/samples/core/loop_output/loop_output.py b/samples/core/loop_output/loop_output.py index 4c3441fcb31..2ff75e5fa9c 100644 --- a/samples/core/loop_output/loop_output.py +++ b/samples/core/loop_output/loop_output.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp import components -from kfp import dsl +from kfp.deprecated import components +from kfp.deprecated import dsl @components.create_component_from_func diff --git a/samples/core/loop_output/loop_output_test.py b/samples/core/loop_output/loop_output_test.py index 32f3d0da85d..b9b49ae9cae 100644 --- a/samples/core/loop_output/loop_output_test.py +++ b/samples/core/loop_output/loop_output_test.py @@ -15,7 +15,7 @@ from __future__ import annotations import unittest -import kfp +import kfp.deprecated as kfp import kfp_server_api from ml_metadata.proto import Execution from .loop_output import my_pipeline diff --git a/samples/core/loop_parallelism/loop_parallelism.py b/samples/core/loop_parallelism/loop_parallelism.py index 118d2588572..18e4853c74a 100644 --- a/samples/core/loop_parallelism/loop_parallelism.py +++ b/samples/core/loop_parallelism/loop_parallelism.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -import kfp +from kfp.deprecated import dsl, components, compiler - -@kfp.components.create_component_from_func +@components.create_component_from_func def print_op(s: str): print(s) @@ -30,4 +28,4 @@ def pipeline(): if __name__ == '__main__': - kfp.compiler.Compiler().compile(pipeline, __file__ + '.yaml') + compiler.Compiler().compile(pipeline, __file__ + '.yaml') diff --git a/samples/core/loop_parallelism/loop_parallelism_test.py b/samples/core/loop_parallelism/loop_parallelism_test.py index 5632769436f..09835ed6160 100644 --- a/samples/core/loop_parallelism/loop_parallelism_test.py +++ b/samples/core/loop_parallelism/loop_parallelism_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .loop_parallelism import pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/core/loop_parameter/loop_parameter.py b/samples/core/loop_parameter/loop_parameter.py index 8502e038f87..1c5c786ed2b 100644 --- a/samples/core/loop_parameter/loop_parameter.py +++ b/samples/core/loop_parameter/loop_parameter.py @@ -1,5 +1,4 @@ -from kfp import components -from kfp import dsl +from kfp.deprecated import components, dsl from typing import List diff --git a/samples/core/loop_parameter/loop_parameter_test.py b/samples/core/loop_parameter/loop_parameter_test.py index 7d8af31190b..74c49622e4d 100644 --- a/samples/core/loop_parameter/loop_parameter_test.py +++ b/samples/core/loop_parameter/loop_parameter_test.py @@ -14,7 +14,7 @@ from __future__ import annotations import unittest -import kfp +import kfp.deprecated as kfp import kfp_server_api from .loop_parameter import my_pipeline from .loop_parameter_v2 import my_pipeline as my_pipeline_v2 diff --git a/samples/core/loop_static/loop_static.py b/samples/core/loop_static/loop_static.py index 3b277397cf5..202869338e0 100644 --- a/samples/core/loop_static/loop_static.py +++ b/samples/core/loop_static/loop_static.py @@ -1,5 +1,4 @@ -from kfp import components -from kfp import dsl +from kfp.deprecated import components, dsl from typing import List diff --git a/samples/core/loop_static/loop_static_test.py b/samples/core/loop_static/loop_static_test.py index 29d99faa125..c11ad9312e2 100644 --- a/samples/core/loop_static/loop_static_test.py +++ b/samples/core/loop_static/loop_static_test.py @@ -14,7 +14,7 @@ from __future__ import annotations import unittest -import kfp +import kfp.deprecated as kfp import kfp_server_api from .loop_static import my_pipeline from .loop_static_v2 import my_pipeline as my_pipeline_v2 diff --git a/samples/core/multiple_outputs/multiple_outputs.ipynb b/samples/core/multiple_outputs/multiple_outputs.ipynb index 6c3ba732974..d15b6da3a43 100644 --- a/samples/core/multiple_outputs/multiple_outputs.ipynb +++ b/samples/core/multiple_outputs/multiple_outputs.ipynb @@ -48,9 +48,9 @@ "metadata": {}, "outputs": [], "source": [ - "import kfp \n", - "import kfp.components as components\n", - "import kfp.dsl as dsl\n", + "import kfp.deprecated as kfp\n", + "import kfp.deprecated.components as components\n", + "import kfp.deprecated.dsl as dsl\n", "from typing import NamedTuple" ] }, diff --git a/samples/core/multiple_outputs/multiple_outputs_test.py b/samples/core/multiple_outputs/multiple_outputs_test.py index d031f0315e8..8cfcaf17b55 100644 --- a/samples/core/multiple_outputs/multiple_outputs_test.py +++ b/samples/core/multiple_outputs/multiple_outputs_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/parallel_join/parallel_join.py b/samples/core/parallel_join/parallel_join.py index a5f7a3d3309..1dcf623f706 100755 --- a/samples/core/parallel_join/parallel_join.py +++ b/samples/core/parallel_join/parallel_join.py @@ -14,7 +14,6 @@ # limitations under the License. -import kfp.deprecated as kfp from kfp.deprecated import dsl, compiler def gcs_download_op(url): diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py index 347788154e0..e14bfc4b141 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py @@ -16,8 +16,8 @@ import json import os -import kfp -import kfp.onprem +import kfp.deprecated as kfp +import kfp.deprecated.onprem import tfx.orchestration.kubeflow.kubeflow_dag_runner as kubeflow_dag_runner import tensorflow_model_analysis as tfma from tfx import v1 as tfx @@ -168,7 +168,7 @@ def _create_pipeline( tfx_image='gcr.io/tfx-oss-public/tfx:%s' % tfx.__version__, pipeline_operator_funcs=kubeflow_dag_runner .get_default_pipeline_operator_funcs(use_gcp_sa=False) + [ - kfp.onprem.add_default_resource_spec( + kfp.deprecated.onprem.add_default_resource_spec( memory_limit='2Gi', cpu_limit='2', cpu_request='1'), ], ) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py index ddffcae130d..3293bbb5594 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py @@ -13,7 +13,7 @@ # limitations under the License. import json -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func bucket = 'kfp-ci' diff --git a/samples/core/pipeline_parallelism/pipeline_parallelism_limits.py b/samples/core/pipeline_parallelism/pipeline_parallelism_limits.py index 1e475ec74ab..e4e62f45685 100644 --- a/samples/core/pipeline_parallelism/pipeline_parallelism_limits.py +++ b/samples/core/pipeline_parallelism/pipeline_parallelism_limits.py @@ -14,12 +14,10 @@ # limitations under the License. -import kfp -from kfp import dsl -import kfp.components as comp +from kfp.deprecated import dsl, compiler, components -@comp.create_component_from_func +@components.create_component_from_func def print_op(msg): """Print a message.""" print(msg) @@ -35,4 +33,4 @@ def pipeline_parallelism(): dsl.get_pipeline_conf().set_parallelism(1) if __name__ == '__main__': - kfp.compiler.Compiler().compile(pipeline_parallelism, __file__ + '.yaml') + compiler.Compiler().compile(pipeline_parallelism, __file__ + '.yaml') diff --git a/samples/core/pipeline_transformers/pipeline_transformers.py b/samples/core/pipeline_transformers/pipeline_transformers.py index ba774e23df0..ec70ea8df10 100644 --- a/samples/core/pipeline_transformers/pipeline_transformers.py +++ b/samples/core/pipeline_transformers/pipeline_transformers.py @@ -14,9 +14,8 @@ # limitations under the License. -import kfp -from kfp import dsl -import kfp.components as comp +import kfp.deprecated.components as comp +from kfp.deprecated import dsl, compiler @comp.create_component_from_func @@ -40,4 +39,4 @@ def transform_pipeline(): dsl.get_pipeline_conf().add_op_transformer(add_annotation) if __name__ == '__main__': - kfp.compiler.Compiler().compile(transform_pipeline, __file__ + '.yaml') + compiler.Compiler().compile(transform_pipeline, __file__ + '.yaml') diff --git a/samples/core/pipeline_transformers/pipeline_transformers_test.py b/samples/core/pipeline_transformers/pipeline_transformers_test.py index ce8bc624160..2fd19033844 100644 --- a/samples/core/pipeline_transformers/pipeline_transformers_test.py +++ b/samples/core/pipeline_transformers/pipeline_transformers_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu.py b/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu.py index 47278fe7fce..fd88e1e5e2c 100644 --- a/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu.py +++ b/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu.py @@ -14,8 +14,7 @@ # limitations under the License. """This is a simple toy example demonstrating how to use preemptible computing resources.""" -from kfp import dsl -from kfp import gcp +from kfp.deprecated import dsl, compiler, gcp class FlipCoinOp(dsl.ContainerOp): @@ -41,5 +40,4 @@ def flipcoin(): if __name__ == '__main__': - import kfp.compiler as compiler compiler.Compiler().compile(flipcoin, __file__ + '.yaml') diff --git a/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu_test.py b/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu_test.py index 76121b4e014..09460bd03ea 100644 --- a/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu_test.py +++ b/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/recursion/recursion.py b/samples/core/recursion/recursion.py index 3b8094f5dfa..24ca27fcf5b 100644 --- a/samples/core/recursion/recursion.py +++ b/samples/core/recursion/recursion.py @@ -13,11 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Notice: caching is tricky when recursion is involved. Please be careful and +# Notice: caching is tricky when recursion is involved. Please be careful and # set proper max_cache_staleness in case of infinite loop. -import kfp -from kfp import dsl +from kfp.deprecated import dsl, compiler def flip_coin_op(): @@ -72,4 +71,4 @@ def flipcoin(): if __name__ == '__main__': - kfp.compiler.Compiler().compile(flipcoin, __file__ + '.yaml') + compiler.Compiler().compile(flipcoin, __file__ + '.yaml') diff --git a/samples/core/resource_ops/resource_ops.py b/samples/core/resource_ops/resource_ops.py index de889d9e17c..873fee5f919 100644 --- a/samples/core/resource_ops/resource_ops.py +++ b/samples/core/resource_ops/resource_ops.py @@ -18,8 +18,7 @@ """ import json -import kfp -import kfp.dsl as dsl +from kfp.deprecated import dsl, compiler _CONTAINER_MANIFEST = """ @@ -43,7 +42,7 @@ "restartPolicy": "Never" } }, - "backoffLimit": 4 + "backoffLimit": 4 } } """ @@ -64,4 +63,4 @@ def resourceop_basic(): if __name__ == '__main__': - kfp.compiler.Compiler().compile(resourceop_basic, __file__ + '.yaml') + compiler.Compiler().compile(resourceop_basic, __file__ + '.yaml') diff --git a/samples/core/resource_ops/resource_ops_test.py b/samples/core/resource_ops/resource_ops_test.py index fb2abfca0a0..90fd9ea1568 100644 --- a/samples/core/resource_ops/resource_ops_test.py +++ b/samples/core/resource_ops/resource_ops_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/resource_spec/resource_spec.py b/samples/core/resource_spec/resource_spec.py index 90acea5e222..85454a794f6 100644 --- a/samples/core/resource_spec/resource_spec.py +++ b/samples/core/resource_spec/resource_spec.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp import components -from kfp import dsl +from kfp.deprecated import components +from kfp.deprecated import dsl @components.create_component_from_func diff --git a/samples/core/resource_spec/resource_spec_test.py b/samples/core/resource_spec/resource_spec_test.py index 5d07413da31..71361999cb1 100644 --- a/samples/core/resource_spec/resource_spec_test.py +++ b/samples/core/resource_spec/resource_spec_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .resource_spec import my_pipeline from .resource_spec_v2 import my_pipeline as my_pipeline_v2 from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/core/resource_spec/runtime_resource_request.py b/samples/core/resource_spec/runtime_resource_request.py index b667451592d..4be0556634a 100644 --- a/samples/core/resource_spec/runtime_resource_request.py +++ b/samples/core/resource_spec/runtime_resource_request.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp import dsl, components +from kfp.deprecated import dsl, components, compiler from typing import NamedTuple @components.create_component_from_func @@ -26,7 +25,7 @@ def training_op(n: int) -> int: def generate_resource_request() -> NamedTuple('output', [('memory', str), ('cpu', str)]): '''Returns the memory and cpu request''' from collections import namedtuple - + resource_output = namedtuple('output', ['memory', 'cpu']) return resource_output('500Mi', '200m') @@ -45,4 +44,4 @@ def resource_request_pipeline(n: int = 11234567): traning_task.execution_options.caching_strategy.max_cache_staleness = 'P0D' if __name__ == '__main__': - kfp.compiler.Compiler().compile(resource_request_pipeline, __file__ + '.yaml') + compiler.Compiler().compile(resource_request_pipeline, __file__ + '.yaml') diff --git a/samples/core/resource_spec/runtime_resource_request_gpu.py b/samples/core/resource_spec/runtime_resource_request_gpu.py index 5aad06c487b..6e01b449e21 100644 --- a/samples/core/resource_spec/runtime_resource_request_gpu.py +++ b/samples/core/resource_spec/runtime_resource_request_gpu.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp import dsl, components +from kfp.deprecated import dsl, components, compiler from typing import NamedTuple @@ -52,4 +51,4 @@ def resource_constraint_request_pipeline(): traning_task.execution_options.caching_strategy.max_cache_staleness = 'P0D' if __name__ == '__main__': - kfp.compiler.Compiler().compile(resource_constraint_request_pipeline, __file__ + '.yaml') + compiler.Compiler().compile(resource_constraint_request_pipeline, __file__ + '.yaml') diff --git a/samples/core/resource_spec/runtime_resource_request_test.py b/samples/core/resource_spec/runtime_resource_request_test.py index a9d5835b246..28aa8bf3562 100644 --- a/samples/core/resource_spec/runtime_resource_request_test.py +++ b/samples/core/resource_spec/runtime_resource_request_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .runtime_resource_request import resource_request_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/core/retry/retry.py b/samples/core/retry/retry.py index f94bdfcb01b..3af7c7fc07e 100755 --- a/samples/core/retry/retry.py +++ b/samples/core/retry/retry.py @@ -14,9 +14,8 @@ # limitations under the License. -import kfp -from kfp import dsl -import kfp.components as comp +from kfp.deprecated import dsl, compiler +import kfp.deprecated.components as comp @comp.create_component_from_func @@ -40,4 +39,4 @@ def retry_sample_pipeline(): if __name__ == '__main__': - kfp.compiler.Compiler().compile(retry_sample_pipeline, __file__ + '.yaml') + compiler.Compiler().compile(retry_sample_pipeline, __file__ + '.yaml') diff --git a/samples/core/retry/retry_test.py b/samples/core/retry/retry_test.py index 8faa51d2d26..cfe017d4436 100644 --- a/samples/core/retry/retry_test.py +++ b/samples/core/retry/retry_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/secret/secret.py b/samples/core/secret/secret.py index 617ea35f5d8..d1193516041 100644 --- a/samples/core/secret/secret.py +++ b/samples/core/secret/secret.py @@ -14,8 +14,7 @@ # limitations under the License. -import kfp -from kfp import dsl +from kfp.deprecated import dsl, compiler # Accessing GCS using the Google Cloud SDK command-line programs @@ -69,4 +68,4 @@ def secret_op_pipeline( gcs_list_buckets_task = gcs_list_buckets_op() if __name__ == '__main__': - kfp.compiler.Compiler().compile(secret_op_pipeline, __file__ + '.yaml') + compiler.Compiler().compile(secret_op_pipeline, __file__ + '.yaml') diff --git a/samples/core/secret/secret_test.py b/samples/core/secret/secret_test.py index 95580f91446..69641ca3c91 100644 --- a/samples/core/secret/secret_test.py +++ b/samples/core/secret/secret_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/sequential/sequential.py b/samples/core/sequential/sequential.py index 6c39fb2937d..f69a9af198d 100755 --- a/samples/core/sequential/sequential.py +++ b/samples/core/sequential/sequential.py @@ -14,8 +14,7 @@ # limitations under the License. -import kfp -from kfp import dsl +from kfp.deprecated import dsl, compiler def gcs_download_op(url): @@ -49,4 +48,4 @@ def sequential_pipeline(url='gs://ml-pipeline/sample-data/shakespeare/shakespear echo_task = echo_op(download_task.output) if __name__ == '__main__': - kfp.compiler.Compiler().compile(sequential_pipeline, __file__ + '.yaml') + compiler.Compiler().compile(sequential_pipeline, __file__ + '.yaml') diff --git a/samples/core/sidecar/sidecar.py b/samples/core/sidecar/sidecar.py index edf84479417..10d07ad2011 100644 --- a/samples/core/sidecar/sidecar.py +++ b/samples/core/sidecar/sidecar.py @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -import kfp.dsl as dsl +from kfp.deprecated import dsl, compiler @dsl.pipeline( @@ -44,4 +43,4 @@ def pipeline_with_sidecar(): if __name__ == '__main__': - kfp.compiler.Compiler().compile(pipeline_with_sidecar, __file__ + '.yaml') + compiler.Compiler().compile(pipeline_with_sidecar, __file__ + '.yaml') diff --git a/samples/core/sidecar/sidecar_test.py b/samples/core/sidecar/sidecar_test.py index d56018bc9d2..ad7177f454d 100644 --- a/samples/core/sidecar/sidecar_test.py +++ b/samples/core/sidecar/sidecar_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/train_until_good/train_until_good.py b/samples/core/train_until_good/train_until_good.py index 3e13c373e8a..b85b8ed5715 100644 --- a/samples/core/train_until_good/train_until_good.py +++ b/samples/core/train_until_good/train_until_good.py @@ -17,8 +17,8 @@ # The main pipeline trains the initial model and then gradually trains the model # some more until the model evaluation metrics are good enough. -import kfp -from kfp import components +import kfp.deprecated as kfp +from kfp.deprecated import components chicago_taxi_dataset_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml') @@ -50,7 +50,7 @@ def train_until_low_error(starting_model, training_data, true_values): label_column=0, ).output - # Calculating the regression metrics + # Calculating the regression metrics metrics_task = calculate_regression_metrics_from_csv_op( true_values=true_values, predicted_values=predictions, @@ -81,7 +81,7 @@ def train_until_good_pipeline(): table=training_data, transform_code='df = df[["tips"]]', ).output - + true_values = drop_header_op(true_values_table).output # Initial model training diff --git a/samples/core/use_run_info/use_run_id.py b/samples/core/use_run_info/use_run_id.py index 41eea3d60a5..d1adc378e10 100644 --- a/samples/core/use_run_info/use_run_id.py +++ b/samples/core/use_run_info/use_run_id.py @@ -13,25 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -import kfp.dsl as dsl +from kfp.deprecated import dsl, compiler, Client, components def get_run_info(run_id: str): """Example of getting run info for current pipeline run.""" - import kfp print(f'Current run ID is {run_id}.') # KFP API server is usually available as ml-pipeline service in the same # namespace, but for full Kubeflow deployment, you need to edit this to # http://ml-pipeline.kubeflow:8888, because your pipelines are running in # user namespaces, but the API is at kubeflow namespace. - client = kfp.Client(host='http://ml-pipeline:8888') + client = Client(host='http://ml-pipeline:8888') run_info = client.get_run(run_id=run_id) # Hide verbose info print(run_info.run) -get_run_info_component = kfp.components.create_component_from_func( +get_run_info_component = components.create_component_from_func( func=get_run_info, packages_to_install=['kfp'], ) @@ -41,11 +39,11 @@ def get_run_info(run_id: str): name='use-run-id', description='A pipeline that demonstrates how to use run information, including run ID etc.' ) -def pipeline_use_run_id(run_id: str = kfp.dsl.RUN_ID_PLACEHOLDER): +def pipeline_use_run_id(run_id: str = dsl.RUN_ID_PLACEHOLDER): """kfp.dsl.RUN_ID_PLACEHOLDER inside a pipeline parameter will be populated with KFP Run ID at runtime.""" run_info_op = get_run_info_component(run_id=run_id) if __name__ == '__main__': - kfp.compiler.Compiler().compile(pipeline_use_run_id, __file__ + '.yaml') + compiler.Compiler().compile(pipeline_use_run_id, __file__ + '.yaml') diff --git a/samples/core/use_run_info/use_run_id_test.py b/samples/core/use_run_info/use_run_id_test.py index 01f1e2fe1f0..a26c3cb47a1 100644 --- a/samples/core/use_run_info/use_run_id_test.py +++ b/samples/core/use_run_info/use_run_id_test.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .use_run_id import pipeline_use_run_id from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/core/visualization/confusion_matrix.py b/samples/core/visualization/confusion_matrix.py index d16057c1218..1716d620f2f 100644 --- a/samples/core/visualization/confusion_matrix.py +++ b/samples/core/visualization/confusion_matrix.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.components import create_component_from_func +import kfp.deprecated.dsl as dsl +from kfp.deprecated.components import create_component_from_func # Advanced function # Demonstrates imports, helper functions and multiple outputs @@ -24,7 +24,7 @@ def confusion_visualization(matrix_uri: str = 'https://raw.githubusercontent.com/kubeflow/pipelines/master/samples/core/visualization/confusion_matrix.csv') -> NamedTuple('VisualizationOutput', [('mlpipeline_ui_metadata', 'UI_metadata')]): """Provide confusion matrix csv file to visualize as metrics.""" import json - + metadata = { 'outputs' : [{ 'type': 'confusion_matrix', @@ -38,12 +38,12 @@ def confusion_visualization(matrix_uri: str = 'https://raw.githubusercontent.com 'labels': ['rose', 'lily', 'iris'], }] } - + from collections import namedtuple visualization_output = namedtuple('VisualizationOutput', [ 'mlpipeline_ui_metadata']) return visualization_output(json.dumps(metadata)) - + @dsl.pipeline( name='confusion-matrix-pipeline', description='A sample pipeline to generate Confusion Matrix for UI visualization.' diff --git a/samples/core/visualization/confusion_matrix_test.py b/samples/core/visualization/confusion_matrix_test.py index fa5592e263a..ef39829f564 100644 --- a/samples/core/visualization/confusion_matrix_test.py +++ b/samples/core/visualization/confusion_matrix_test.py @@ -18,7 +18,7 @@ from .confusion_matrix import confusion_matrix_pipeline, confusion_visualization from kfp.samples.test.utils import KfpMlmdClient, run_pipeline_func, TestCase -import kfp +import kfp.deprecated as kfp def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): diff --git a/samples/core/visualization/html.py b/samples/core/visualization/html.py index 9f09672f5e2..409c83cbf12 100644 --- a/samples/core/visualization/html.py +++ b/samples/core/visualization/html.py @@ -12,14 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.components import create_component_from_func +from kfp.deprecated import dsl, components # Advanced function # Demonstrates imports, helper functions and multiple outputs from typing import NamedTuple -@create_component_from_func +@components.create_component_from_func def html_visualization(gcsPath: str) -> NamedTuple('VisualizationOutput', [('mlpipeline_ui_metadata', 'UI_metadata')]): import json diff --git a/samples/core/visualization/html_test.py b/samples/core/visualization/html_test.py index e058f60599b..2469400185d 100644 --- a/samples/core/visualization/html_test.py +++ b/samples/core/visualization/html_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from .html import html_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase diff --git a/samples/core/visualization/markdown.py b/samples/core/visualization/markdown.py index 9c93da32634..31564d7a9ba 100644 --- a/samples/core/visualization/markdown.py +++ b/samples/core/visualization/markdown.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -import kfp.components as comp +import kfp.deprecated.dsl as dsl +import kfp.deprecated.components as comp # Advanced function # Demonstrates imports, helper functions and multiple outputs diff --git a/samples/core/visualization/markdown_test.py b/samples/core/visualization/markdown_test.py index 1fc82ee51dd..629df3191e9 100644 --- a/samples/core/visualization/markdown_test.py +++ b/samples/core/visualization/markdown_test.py @@ -15,7 +15,7 @@ from .markdown import markdown_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase -import kfp +import kfp.deprecated as kfp run_pipeline_func([ TestCase( diff --git a/samples/core/visualization/roc.py b/samples/core/visualization/roc.py index 7c54d407c7a..b5da899e7f4 100644 --- a/samples/core/visualization/roc.py +++ b/samples/core/visualization/roc.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.components import create_component_from_func +import kfp.deprecated.dsl as dsl +from kfp.deprecated.components import create_component_from_func # Advanced function # Demonstrates imports, helper functions and multiple outputs diff --git a/samples/core/visualization/roc_test.py b/samples/core/visualization/roc_test.py index 03335c008ed..27b4330387e 100644 --- a/samples/core/visualization/roc_test.py +++ b/samples/core/visualization/roc_test.py @@ -18,7 +18,7 @@ from .roc import roc_curve_pipeline from kfp.samples.test.utils import KfpMlmdClient, run_pipeline_func, TestCase -import kfp +import kfp.deprecated as kfp def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): diff --git a/samples/core/visualization/table.py b/samples/core/visualization/table.py index 639cfa0eb19..72d06ce7388 100644 --- a/samples/core/visualization/table.py +++ b/samples/core/visualization/table.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.dsl as dsl -from kfp.components import create_component_from_func +import kfp.deprecated.dsl as dsl +from kfp.deprecated.components import create_component_from_func # Advanced function # Demonstrates imports, helper functions and multiple outputs @@ -24,7 +24,7 @@ def table_visualization(train_file_path: str = 'https://raw.githubusercontent.com/zijianjoy/pipelines/5651f41071816594b2ed27c88367f5efb4c60b50/samples/core/visualization/table.csv') -> NamedTuple('VisualizationOutput', [('mlpipeline_ui_metadata', 'UI_metadata')]): """Provide number to visualize as table metrics.""" import json - + header = ['Average precision ', 'Precision', 'Recall'] metadata = { 'outputs' : [{ diff --git a/samples/core/visualization/table_test.py b/samples/core/visualization/table_test.py index 514b729da09..5962cf06a3c 100644 --- a/samples/core/visualization/table_test.py +++ b/samples/core/visualization/table_test.py @@ -18,7 +18,7 @@ from .table import table_pipeline from kfp.samples.test.utils import KfpMlmdClient, run_pipeline_func, TestCase -import kfp +import kfp.deprecated as kfp def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, diff --git a/samples/core/visualization/tensorboard_gcs.py b/samples/core/visualization/tensorboard_gcs.py index 8e4606a61b2..a347828ce4b 100644 --- a/samples/core/visualization/tensorboard_gcs.py +++ b/samples/core/visualization/tensorboard_gcs.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp import dsl, components -from kfp.components import create_component_from_func +from kfp.deprecated import dsl, components +from kfp.deprecated.components import create_component_from_func prepare_tensorboard = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/1.5.0/components/tensorflow/tensorboard/prepare_tensorboard/component.yaml' diff --git a/samples/core/visualization/tensorboard_minio.py b/samples/core/visualization/tensorboard_minio.py index 16c2e8bc224..db989a8bff6 100644 --- a/samples/core/visualization/tensorboard_minio.py +++ b/samples/core/visualization/tensorboard_minio.py @@ -14,9 +14,9 @@ import os import json -from kfp.onprem import use_k8s_secret -from kfp import dsl, components -from kfp.components import OutputPath, create_component_from_func +from kfp.deprecated.onprem import use_k8s_secret +from kfp.deprecated import dsl, components +from kfp.deprecated.components import OutputPath, create_component_from_func prepare_tensorboard = components.load_component_from_url( diff --git a/samples/core/visualization/tensorboard_minio_test.py b/samples/core/visualization/tensorboard_minio_test.py index 43cb8fa74d0..0a2a3d78f5f 100644 --- a/samples/core/visualization/tensorboard_minio_test.py +++ b/samples/core/visualization/tensorboard_minio_test.py @@ -15,7 +15,7 @@ import unittest import json from pprint import pprint -import kfp +import kfp.deprecated as kfp import kfp_server_api from google.cloud import storage diff --git a/samples/core/volume_ops/volume_ops.py b/samples/core/volume_ops/volume_ops.py index 91652fa4f37..05bdee7488e 100644 --- a/samples/core/volume_ops/volume_ops.py +++ b/samples/core/volume_ops/volume_ops.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -import kfp.dsl as dsl -from kfp import components +from kfp.deprecated import components, dsl, compiler @components.create_component_from_func def write_to_volume(): @@ -33,9 +31,9 @@ def volumeop_basic(size: str="1Gi"): modes=dsl.VOLUME_MODE_RWO, size=size ) - + write_to_volume().add_pvolumes({"/mnt'": vop.volume}) if __name__ == '__main__': - kfp.compiler.Compiler().compile(volumeop_basic, __file__ + '.yaml') + compiler.Compiler().compile(volumeop_basic, __file__ + '.yaml') diff --git a/samples/core/volume_snapshot_ops/volume_snapshot_ops.py b/samples/core/volume_snapshot_ops/volume_snapshot_ops.py index 74e4f188be1..8a6609e54b0 100644 --- a/samples/core/volume_snapshot_ops/volume_snapshot_ops.py +++ b/samples/core/volume_snapshot_ops/volume_snapshot_ops.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -import kfp.dsl as dsl +from kfp.deprecated import dsl, compiler @dsl.pipeline( @@ -81,4 +80,4 @@ def volume_snapshotop_sequential(url): ) if __name__ == '__main__': - kfp.compiler.Compiler().compile(volume_snapshotop_sequential, __file__ + '.yaml') + compiler.Compiler().compile(volume_snapshotop_sequential, __file__ + '.yaml') diff --git a/samples/core/volume_snapshot_ops/volume_snapshot_ops_test.py b/samples/core/volume_snapshot_ops/volume_snapshot_ops_test.py index c62b43a4fe8..148f38c34ac 100644 --- a/samples/core/volume_snapshot_ops/volume_snapshot_ops_test.py +++ b/samples/core/volume_snapshot_ops/volume_snapshot_ops_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ diff --git a/samples/core/xgboost_training_cm/xgboost_training_cm.py b/samples/core/xgboost_training_cm/xgboost_training_cm.py index 678e3fab117..2e8de67ee07 100644 --- a/samples/core/xgboost_training_cm/xgboost_training_cm.py +++ b/samples/core/xgboost_training_cm/xgboost_training_cm.py @@ -15,9 +15,8 @@ import json -import kfp -from kfp import components -from kfp import dsl +from kfp.deprecated import dsl, compiler, components + import os import subprocess @@ -229,14 +228,14 @@ def xgb_train_pipeline( transform_output_eval = os.path.join(output_template, 'eval', 'part-*') train_output = os.path.join(output_template, 'train_output') predict_output = os.path.join(output_template, 'predict_output') - + _diagnose_me_op = diagnose_me_op( bucket=output, execution_mode=diagnostic_mode, - project_id=project, + project_id=project, target_apis=required_apis, quota_check=quota_check) - + with dsl.ExitHandler(exit_op=dataproc_delete_cluster_op( project_id=project, region=region, @@ -310,4 +309,4 @@ def xgb_train_pipeline( ).after(_predict_op) if __name__ == '__main__': - kfp.compiler.Compiler().compile(xgb_train_pipeline, __file__ + '.yaml') + compiler.Compiler().compile(xgb_train_pipeline, __file__ + '.yaml') From cfe43475e725794b49abcd147caca14f75915f28 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 16 Feb 2022 00:31:53 -0800 Subject: [PATCH 17/39] add readme --- sdk/RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/RELEASE.md b/sdk/RELEASE.md index b8f450b0eec..54352a68ad3 100644 --- a/sdk/RELEASE.md +++ b/sdk/RELEASE.md @@ -19,6 +19,7 @@ * APIs imported from the v1 namespace are no longer supported by the v2 compiler. [\#6890](https://github.com/kubeflow/pipelines/pull/6890) * Deprecate v2 compatible mode in v1 compiler. [\#6958](https://github.com/kubeflow/pipelines/pull/6958) * Drop support for python 3.6 [\#7303](https://github.com/kubeflow/pipelines/pull/7303) +* Deprecate v1 code to deprecated folder [\#7291](https://github.com/kubeflow/pipelines/pull/7291) ### For Pipeline Authors From fca4351054a24f63671f53c0133847a31fb44b3d Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 16 Feb 2022 00:48:27 -0800 Subject: [PATCH 18/39] restroe test --- .../test_data/pipeline_with_task_final_status.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_task_final_status.json b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_task_final_status.json index cef8d1994fc..b3a522a24c5 100644 --- a/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_task_final_status.json +++ b/sdk/python/kfp/v2/compiler_cli_tests/test_data/pipeline_with_task_final_status.json @@ -118,7 +118,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.11' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", @@ -138,7 +138,7 @@ "command": [ "sh", "-c", - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.11' && \"$0\" \"$@\"\n", + "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0b0' && \"$0\" \"$@\"\n", "sh", "-ec", "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", From bede848f7670edad140e0daad68648f96d6e77a8 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 16 Feb 2022 09:54:32 -0800 Subject: [PATCH 19/39] python require --- samples/test/utils/kfp/samples/test/utils.py | 2 +- samples/test/utils/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/test/utils/kfp/samples/test/utils.py b/samples/test/utils/kfp/samples/test/utils.py index 021e24c798c..03a95b979c6 100644 --- a/samples/test/utils/kfp/samples/test/utils.py +++ b/samples/test/utils/kfp/samples/test/utils.py @@ -76,7 +76,7 @@ class TestCase: pipeline_func: Optional[Callable] = None pipeline_file: Optional[str] = None pipeline_file_compile_path: Optional[str] = None - mode: kfp.dsl.PipelineExecutionMode = kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE + mode: kfp.dsl.PipelineExecutionMode = kfp.dsl.PipelineExecutionMode.V1_LEGACY enable_caching: bool = False arguments: Optional[dict[str, str]] = None verify_func: Verifier = _default_verify_func diff --git a/samples/test/utils/setup.py b/samples/test/utils/setup.py index 102771d1c63..a90dea67434 100644 --- a/samples/test/utils/setup.py +++ b/samples/test/utils/setup.py @@ -36,5 +36,5 @@ ], package_dir={"": "."}, packages=setuptools.find_packages(where="."), - python_requires=">=3.6", + python_requires=">=3.7", ) From be082b464d433c654f68e0a45195b367d65c2672 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 16 Feb 2022 16:56:54 -0800 Subject: [PATCH 20/39] remove path --- sdk/python/kfp/deprecated/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sdk/python/kfp/deprecated/__init__.py b/sdk/python/kfp/deprecated/__init__.py index c6d2d84f83d..0e2b122c56f 100644 --- a/sdk/python/kfp/deprecated/__init__.py +++ b/sdk/python/kfp/deprecated/__init__.py @@ -12,12 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# `kfp` is a namespace package. -# https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages -__path__ = __import__("pkgutil").extend_path(__path__, __name__) - -__version__ = '1.8.11' - from . import components from . import containers from . import dsl From 64110c5cb52e1ff5c6a7d4868ad30dcc1b8afe4f Mon Sep 17 00:00:00 2001 From: Yaqi Date: Mon, 21 Feb 2022 16:13:17 -0800 Subject: [PATCH 21/39] fix tests --- samples/test/utils/kfp/samples/test/utils.py | 4 +- sdk/python/kfp/deprecated/__init__.py | 2 + sdk/python/kfp/deprecated/compiler/main.py | 1 - sdk/python/kfp/deprecated/dsl/_component.py | 1 - sdk/python/kfp/deprecated/notebook/_magic.py | 42 -------------------- 5 files changed, 4 insertions(+), 46 deletions(-) diff --git a/samples/test/utils/kfp/samples/test/utils.py b/samples/test/utils/kfp/samples/test/utils.py index 03a95b979c6..7bb98ea3392 100644 --- a/samples/test/utils/kfp/samples/test/utils.py +++ b/samples/test/utils/kfp/samples/test/utils.py @@ -30,7 +30,7 @@ import nbformat from nbconvert import PythonExporter -import kfp.deprecated as kfp +import kfp from kfp.deprecated.onprem import add_default_resource_spec import kfp.v2.compiler import kfp_server_api @@ -76,7 +76,7 @@ class TestCase: pipeline_func: Optional[Callable] = None pipeline_file: Optional[str] = None pipeline_file_compile_path: Optional[str] = None - mode: kfp.dsl.PipelineExecutionMode = kfp.dsl.PipelineExecutionMode.V1_LEGACY + mode: kfp.deprecated.dsl.PipelineExecutionMode = kfp.dsl.PipelineExecutionMode.V1_LEGACY enable_caching: bool = False arguments: Optional[dict[str, str]] = None verify_func: Verifier = _default_verify_func diff --git a/sdk/python/kfp/deprecated/__init__.py b/sdk/python/kfp/deprecated/__init__.py index 0e2b122c56f..83acc09ba2d 100644 --- a/sdk/python/kfp/deprecated/__init__.py +++ b/sdk/python/kfp/deprecated/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +__version__ = '1.8.11' + from . import components from . import containers from . import dsl diff --git a/sdk/python/kfp/deprecated/compiler/main.py b/sdk/python/kfp/deprecated/compiler/main.py index ef15c21221c..ab20e3d7ad9 100644 --- a/sdk/python/kfp/deprecated/compiler/main.py +++ b/sdk/python/kfp/deprecated/compiler/main.py @@ -19,7 +19,6 @@ import kfp.deprecated.compiler import os import sys -from deprecated.sphinx import deprecated _KF_PIPELINES_COMPILER_MODE_ENV = 'KF_PIPELINES_COMPILER_MODE' diff --git a/sdk/python/kfp/deprecated/dsl/_component.py b/sdk/python/kfp/deprecated/dsl/_component.py index e104641a1d5..92d72edd4fe 100644 --- a/sdk/python/kfp/deprecated/dsl/_component.py +++ b/sdk/python/kfp/deprecated/dsl/_component.py @@ -13,7 +13,6 @@ # limitations under the License. import inspect -from deprecated.sphinx import deprecated from ._pipeline_param import PipelineParam from .types import check_types, InconsistentTypeException from ._ops_group import Graph diff --git a/sdk/python/kfp/deprecated/notebook/_magic.py b/sdk/python/kfp/deprecated/notebook/_magic.py index dedfdc910af..011b6cb6814 100644 --- a/sdk/python/kfp/deprecated/notebook/_magic.py +++ b/sdk/python/kfp/deprecated/notebook/_magic.py @@ -15,46 +15,4 @@ import os import tempfile -from deprecated.sphinx import deprecated - from kfp.deprecated.compiler import build_docker_image - - -@deprecated( - version='0.1.32', - reason='The %%docker magic is deprecated. Use `kfp.containers.build_image_from_working_dir` instead.' -) -def docker(line, cell): - """cell magic for %%docker.""" - - if len(line.split()) < 2: - raise ValueError( - "usage: %%docker [gcr.io/project/image:tag] [gs://staging-bucket] [600] [kubeflow]\n\ - arg1(required): target image tag\n\ - arg2(required): staging gcs bucket\n\ - arg3(optional): timeout in seconds, default(600)\n\ - arg4(optional): namespace, default(kubeflow)") - if not cell.strip(): - raise ValueError("Please fill in a dockerfile content in the cell.") - - fields = line.split() - - with tempfile.NamedTemporaryFile(mode='wt', delete=False) as f: - f.write(cell) - - build_docker_image( - fields[1], - fields[0], - f.name, - timeout=int(fields[2]) if len(fields) >= 3 else 600, - namespace=fields[3] if len(fields) >= 4 else 'kubeflow') - os.remove(f.name) - - -try: - import IPython - docker = IPython.core.magic.register_cell_magic(docker) -except ImportError: - pass -except NameError: - pass From d4678aa862005441b465a7f25f9d5988579210b0 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Mon, 21 Feb 2022 19:02:25 -0800 Subject: [PATCH 22/39] inteegration tests --- samples/test/utils/kfp/samples/test/utils.py | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/samples/test/utils/kfp/samples/test/utils.py b/samples/test/utils/kfp/samples/test/utils.py index 7bb98ea3392..42516becc84 100644 --- a/samples/test/utils/kfp/samples/test/utils.py +++ b/samples/test/utils/kfp/samples/test/utils.py @@ -76,7 +76,7 @@ class TestCase: pipeline_func: Optional[Callable] = None pipeline_file: Optional[str] = None pipeline_file_compile_path: Optional[str] = None - mode: kfp.deprecated.dsl.PipelineExecutionMode = kfp.dsl.PipelineExecutionMode.V1_LEGACY + mode: kfp.deprecated.dsl.PipelineExecutionMode = kfp.deprecated.dsl.PipelineExecutionMode.V1_LEGACY enable_caching: bool = False arguments: Optional[dict[str, str]] = None verify_func: Verifier = _default_verify_func @@ -96,7 +96,7 @@ def run_pipeline_func(test_cases: list[TestCase]): def test_wrapper( run_pipeline: Callable[[ - Callable, str, str, kfp.dsl.PipelineExecutionMode, bool, dict, bool + Callable, str, str, kfp.deprecated.dsl.PipelineExecutionMode, bool, dict, bool ], kfp_server_api.ApiRunDetail], mlmd_connection_config: metadata_store_pb2.MetadataStoreClientConfig, ): @@ -120,14 +120,14 @@ def test_wrapper( 'TestCase.run_pipeline = False can only be specified when used together with pipeline_file.' ) - if case.mode == kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE: + if case.mode == kfp.deprecated.dsl.PipelineExecutionMode.V2_COMPATIBLE: print( f'Unexpected v2 compatible mode test for: {pipeline_name}') raise RuntimeError - if case.mode == kfp.dsl.PipelineExecutionMode.V2_ENGINE: + if case.mode == kfp.deprecated.dsl.PipelineExecutionMode.V2_ENGINE: print(f'Running v2 engine mode test for: {pipeline_name}') - if case.mode == kfp.dsl.PipelineExecutionMode.V1_LEGACY: + if case.mode == kfp.deprecated.dsl.PipelineExecutionMode.V1_LEGACY: print(f'Running v1 legacy test for: {pipeline_name}') run_detail = run_pipeline( @@ -152,7 +152,7 @@ def test_wrapper( tasks = {} client = None # we cannot stably use MLMD to query status in v1, because it may be async. - if case.mode == kfp.dsl.PipelineExecutionMode.V2_ENGINE: + if case.mode == kfp.deprecated.dsl.PipelineExecutionMode.V2_ENGINE: client = KfpMlmdClient( mlmd_connection_config=mlmd_connection_config) tasks = client.get_tasks(run_id=run_detail.run.id) @@ -259,7 +259,7 @@ def main( if not driver_image: raise Exception("driver_image is empty") logger.info(f'KFP_DRIVER_IMAGE={driver_image}') - client = kfp.Client() + client = kfp.deprecated.Client() # TODO(Bobgy): avoid using private fields when getting loaded config kfp_endpoint = client._existing_config.host kfp_ui_endpoint = client._uihost @@ -271,7 +271,7 @@ def run_pipeline( pipeline_func: Optional[Callable], pipeline_file: Optional[str], pipeline_file_compile_path: Optional[str], - mode: kfp.dsl.PipelineExecutionMode = kfp.dsl.PipelineExecutionMode + mode: kfp.deprecated.dsl.PipelineExecutionMode = kfp.deprecated.dsl.PipelineExecutionMode .V2_ENGINE, enable_caching: bool = False, arguments: Optional[dict] = None, @@ -281,7 +281,7 @@ def run_pipeline( arguments = arguments or {} def _create_run(): - if mode == kfp.dsl.PipelineExecutionMode.V2_ENGINE: + if mode == kfp.deprecated.dsl.PipelineExecutionMode.V2_ENGINE: return run_v2_pipeline( client=client, fn=pipeline_func, @@ -295,7 +295,7 @@ def _create_run(): }, ) else: - conf = kfp.dsl.PipelineConf() + conf = kfp.deprecated.dsl.PipelineConf() conf.add_op_transformer( # add a default resource request & limit to all container tasks add_default_resource_spec( @@ -303,7 +303,7 @@ def _create_run(): cpu_limit='1', memory_limit='512Mi', )) - if mode == kfp.dsl.PipelineExecutionMode.V1_LEGACY: + if mode == kfp.deprecated.dsl.PipelineExecutionMode.V1_LEGACY: conf.add_op_transformer(_disable_cache) if pipeline_func: return client.create_run_from_pipeline_func( @@ -377,7 +377,7 @@ def _create_run(): def run_v2_pipeline( - client: kfp.Client, + client: kfp.deprecated.Client, fn: Optional[Callable], file: Optional[str], driver_image: Optional[str], @@ -714,7 +714,7 @@ def _parse_parameters(execution: metadata_store_pb2.Execution) -> dict: def _disable_cache(task): # Skip tasks which are not container ops. - if not isinstance(task, kfp.dsl.ContainerOp): + if not isinstance(task, kfp.deprecated.dsl.ContainerOp): return task task.execution_options.caching_strategy.max_cache_staleness = "P0D" return task From a890239210ecb13fc543c3b6b4d073f40ce6c4b8 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Mon, 21 Feb 2022 21:42:31 -0800 Subject: [PATCH 23/39] remove tfx tests for dependency with kfp v1 --- test/sample_test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/sample_test.yaml b/test/sample_test.yaml index 39cccfa501e..cf5e2b04b67 100644 --- a/test/sample_test.yaml +++ b/test/sample_test.yaml @@ -77,7 +77,8 @@ spec: # - kubeflow_tf_serving # Disable container_build because this approach will be deprecated soon. # - container_build - - parameterized_tfx_oss + # TODO: Re-enable test when tfx switch to v2 + # - parameterized_tfx_oss when: "{{inputs.parameters.is-integration-test}} == true" # Build and push image - name: build-image-by-dockerfile From 986cfb3bfeaacff9bdbb70899b0be2f840502f19 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Mon, 21 Feb 2022 21:45:19 -0800 Subject: [PATCH 24/39] fix e2e --- .../core/train_until_good/train_until_good.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/samples/core/train_until_good/train_until_good.py b/samples/core/train_until_good/train_until_good.py index b85b8ed5715..ffdf0a5bb53 100644 --- a/samples/core/train_until_good/train_until_good.py +++ b/samples/core/train_until_good/train_until_good.py @@ -17,8 +17,8 @@ # The main pipeline trains the initial model and then gradually trains the model # some more until the model evaluation metrics are good enough. -import kfp.deprecated as kfp -from kfp.deprecated import components +import kfp +from kfp.deprecated import components, dsl, Client chicago_taxi_dataset_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml') @@ -26,13 +26,13 @@ xgboost_predict_on_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Predict/component.yaml') pandas_transform_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/6162d55998b176b50267d351241100bb0ee715bc/components/pandas/Transform_DataFrame/in_CSV_format/component.yaml') -drop_header_op = kfp.components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/02c9638287468c849632cf9f7885b51de4c66f86/components/tables/Remove_header/component.yaml') -calculate_regression_metrics_from_csv_op = kfp.components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/616542ac0f789914f4eb53438da713dd3004fba4/components/ml_metrics/Calculate_regression_metrics/from_CSV/component.yaml') +drop_header_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/02c9638287468c849632cf9f7885b51de4c66f86/components/tables/Remove_header/component.yaml') +calculate_regression_metrics_from_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/616542ac0f789914f4eb53438da713dd3004fba4/components/ml_metrics/Calculate_regression_metrics/from_CSV/component.yaml') # This recursive sub-pipeline trains a model, evaluates it, calculates the metrics and checks them. # If the model error is too high, then more training is performed until the model is good. -@kfp.dsl.graph_component +@dsl.graph_component def train_until_low_error(starting_model, training_data, true_values): # Training model = xgboost_train_on_csv_op( @@ -57,7 +57,7 @@ def train_until_low_error(starting_model, training_data, true_values): ) # Checking the metrics - with kfp.dsl.Condition(metrics_task.outputs['mean_squared_error'] > 0.01): + with dsl.Condition(metrics_task.outputs['mean_squared_error'] > 0.01): # Training some more train_until_low_error( starting_model=model, @@ -67,7 +67,7 @@ def train_until_low_error(starting_model, training_data, true_values): # The main pipleine trains the initial model and then gradually trains the model some more until the model evaluation metrics are good enough. -@kfp.dsl.pipeline() +@dsl.pipeline() def train_until_good_pipeline(): # Preparing the training data training_data = chicago_taxi_dataset_op( @@ -102,4 +102,4 @@ def train_until_good_pipeline(): if __name__ == '__main__': kfp_endpoint=None - kfp.Client(host=kfp_endpoint).create_run_from_pipeline_func(train_until_good_pipeline, arguments={}) + Client(host=kfp_endpoint).create_run_from_pipeline_func(train_until_good_pipeline, arguments={}) From 1a2fe34f19e22f9a54dd551b82f400ff395837bb Mon Sep 17 00:00:00 2001 From: Yaqi Date: Mon, 21 Feb 2022 23:01:02 -0800 Subject: [PATCH 25/39] fix e2e --- samples/core/train_until_good/train_until_good.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/samples/core/train_until_good/train_until_good.py b/samples/core/train_until_good/train_until_good.py index ffdf0a5bb53..5948a2bd604 100644 --- a/samples/core/train_until_good/train_until_good.py +++ b/samples/core/train_until_good/train_until_good.py @@ -17,9 +17,7 @@ # The main pipeline trains the initial model and then gradually trains the model # some more until the model evaluation metrics are good enough. -import kfp -from kfp.deprecated import components, dsl, Client - +from kfp import components, dsl, Client chicago_taxi_dataset_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml') xgboost_train_on_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Train/component.yaml') From c844d54655d7c847b686fc1835877e011aab1237 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Mon, 21 Feb 2022 23:06:19 -0800 Subject: [PATCH 26/39] fix integration tests --- .../parameterized_tfx_oss.py | 332 +++++++++--------- .../parameterized_tfx_oss_test.py | 49 +-- samples/test/config-integration.yaml | 5 +- 3 files changed, 195 insertions(+), 191 deletions(-) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py index e14bfc4b141..7c0a77ec811 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py @@ -13,168 +13,170 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import os - -import kfp.deprecated as kfp -import kfp.deprecated.onprem -import tfx.orchestration.kubeflow.kubeflow_dag_runner as kubeflow_dag_runner -import tensorflow_model_analysis as tfma -from tfx import v1 as tfx - -# Define pipeline params used for pipeline execution. -# Path to the module file, should be a GCS path, -# or a module file baked in the docker image used by the pipeline. -_taxi_module_file_param = tfx.dsl.experimental.RuntimeParameter( - name='module-file', - default='/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/taxi_utils_native_keras.py', - ptype=str, -) - -# Path to the CSV data file, under which their should be a data.csv file. -_data_root = '/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/data/simple' - -# Path of pipeline root, should be a GCS path. -_pipeline_root = os.path.join('gs://{{kfp-default-bucket}}', 'tfx_taxi_simple', - kfp.dsl.RUN_ID_PLACEHOLDER) - -# Path that ML models are pushed, should be a GCS path. -_serving_model_dir = os.path.join('gs://your-bucket', 'serving_model', - 'tfx_taxi_simple') -_push_destination = tfx.dsl.experimental.RuntimeParameter( - name='push_destination', - default=json.dumps({'filesystem': { - 'base_directory': _serving_model_dir - }}), - ptype=str, -) - - -def _create_pipeline( - pipeline_root: str, - csv_input_location: str, - taxi_module_file: tfx.dsl.experimental.RuntimeParameter, - push_destination: tfx.dsl.experimental.RuntimeParameter, - enable_cache: bool, -): - """Creates a simple Kubeflow-based Chicago Taxi TFX pipeline. - - Args: - pipeline_root: The root of the pipeline output. - csv_input_location: The location of the input data directory. - taxi_module_file: The location of the module file for Transform/Trainer. - enable_cache: Whether to enable cache or not. - - Returns: - A logical TFX pipeline.Pipeline object. - """ - example_gen = tfx.components.CsvExampleGen(input_base=csv_input_location) - statistics_gen = tfx.components.StatisticsGen( - examples=example_gen.outputs['examples']) - schema_gen = tfx.components.SchemaGen( - statistics=statistics_gen.outputs['statistics'], - infer_feature_shape=False, - ) - example_validator = tfx.components.ExampleValidator( - statistics=statistics_gen.outputs['statistics'], - schema=schema_gen.outputs['schema'], - ) - transform = tfx.components.Transform( - examples=example_gen.outputs['examples'], - schema=schema_gen.outputs['schema'], - module_file=taxi_module_file, - ) - trainer = tfx.components.Trainer( - module_file=taxi_module_file, - examples=transform.outputs['transformed_examples'], - schema=schema_gen.outputs['schema'], - transform_graph=transform.outputs['transform_graph'], - train_args=tfx.proto.TrainArgs(num_steps=10), - eval_args=tfx.proto.EvalArgs(num_steps=5), - ) - # Set the TFMA config for Model Evaluation and Validation. - eval_config = tfma.EvalConfig( - model_specs=[ - tfma.ModelSpec( - signature_name='serving_default', - label_key='tips_xf', - preprocessing_function_names=['transform_features']) - ], - metrics_specs=[ - tfma.MetricsSpec( - # The metrics added here are in addition to those saved with the - # model (assuming either a keras model or EvalSavedModel is used). - # Any metrics added into the saved model (for example using - # model.compile(..., metrics=[...]), etc) will be computed - # automatically. - metrics=[tfma.MetricConfig(class_name='ExampleCount')], - # To add validation thresholds for metrics saved with the model, - # add them keyed by metric name to the thresholds map. - thresholds={ - 'binary_accuracy': - tfma.MetricThreshold( - value_threshold=tfma.GenericValueThreshold( - lower_bound={'value': 0.5}), - change_threshold=tfma.GenericChangeThreshold( - direction=tfma.MetricDirection.HIGHER_IS_BETTER, - absolute={'value': -1e-10})) - }) - ], - slicing_specs=[ - # An empty slice spec means the overall slice, i.e. the whole dataset. - tfma.SlicingSpec(), - # Data can be sliced along a feature column. In this case, data is - # sliced along feature column trip_start_hour. - tfma.SlicingSpec(feature_keys=['trip_start_hour']) - ]) - - evaluator = tfx.components.Evaluator( - examples=example_gen.outputs['examples'], - model=trainer.outputs['model'], - eval_config=eval_config, - ) - - pusher = tfx.components.Pusher( - model=trainer.outputs['model'], - model_blessing=evaluator.outputs['blessing'], - push_destination=push_destination, - ) - - return tfx.dsl.Pipeline( - pipeline_name='parameterized_tfx_oss', - pipeline_root=pipeline_root, - components=[ - example_gen, statistics_gen, schema_gen, example_validator, - transform, trainer, evaluator, pusher - ], - enable_cache=enable_cache, - ) - - -if __name__ == '__main__': - enable_cache = True - pipeline = _create_pipeline( - _pipeline_root, - _data_root, - _taxi_module_file_param, - _push_destination, - enable_cache=enable_cache, - ) - # Make sure the version of TFX image used is consistent with the version of - # TFX SDK. - config = tfx.orchestration.experimental.KubeflowDagRunnerConfig( - kubeflow_metadata_config=tfx.orchestration.experimental - .get_default_kubeflow_metadata_config(), - tfx_image='gcr.io/tfx-oss-public/tfx:%s' % tfx.__version__, - pipeline_operator_funcs=kubeflow_dag_runner - .get_default_pipeline_operator_funcs(use_gcp_sa=False) + [ - kfp.deprecated.onprem.add_default_resource_spec( - memory_limit='2Gi', cpu_limit='2', cpu_request='1'), - ], - ) - kfp_runner = tfx.orchestration.experimental.KubeflowDagRunner( - output_filename=__file__ + '.yaml', - config=config, - ) - - kfp_runner.run(pipeline) +# TODO: TFX oss uses early depencies, uncomment until fix + +# import json +# import os + +# import kfp.deprecated as kfp +# import kfp.deprecated.onprem +# import tfx.orchestration.kubeflow.kubeflow_dag_runner as kubeflow_dag_runner +# import tensorflow_model_analysis as tfma +# from tfx import v1 as tfx + +# # Define pipeline params used for pipeline execution. +# # Path to the module file, should be a GCS path, +# # or a module file baked in the docker image used by the pipeline. +# _taxi_module_file_param = tfx.dsl.experimental.RuntimeParameter( +# name='module-file', +# default='/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/taxi_utils_native_keras.py', +# ptype=str, +# ) + +# # Path to the CSV data file, under which their should be a data.csv file. +# _data_root = '/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/data/simple' + +# # Path of pipeline root, should be a GCS path. +# _pipeline_root = os.path.join('gs://{{kfp-default-bucket}}', 'tfx_taxi_simple', +# kfp.dsl.RUN_ID_PLACEHOLDER) + +# # Path that ML models are pushed, should be a GCS path. +# _serving_model_dir = os.path.join('gs://your-bucket', 'serving_model', +# 'tfx_taxi_simple') +# _push_destination = tfx.dsl.experimental.RuntimeParameter( +# name='push_destination', +# default=json.dumps({'filesystem': { +# 'base_directory': _serving_model_dir +# }}), +# ptype=str, +# ) + + +# def _create_pipeline( +# pipeline_root: str, +# csv_input_location: str, +# taxi_module_file: tfx.dsl.experimental.RuntimeParameter, +# push_destination: tfx.dsl.experimental.RuntimeParameter, +# enable_cache: bool, +# ): +# """Creates a simple Kubeflow-based Chicago Taxi TFX pipeline. + +# Args: +# pipeline_root: The root of the pipeline output. +# csv_input_location: The location of the input data directory. +# taxi_module_file: The location of the module file for Transform/Trainer. +# enable_cache: Whether to enable cache or not. + +# Returns: +# A logical TFX pipeline.Pipeline object. +# """ +# example_gen = tfx.components.CsvExampleGen(input_base=csv_input_location) +# statistics_gen = tfx.components.StatisticsGen( +# examples=example_gen.outputs['examples']) +# schema_gen = tfx.components.SchemaGen( +# statistics=statistics_gen.outputs['statistics'], +# infer_feature_shape=False, +# ) +# example_validator = tfx.components.ExampleValidator( +# statistics=statistics_gen.outputs['statistics'], +# schema=schema_gen.outputs['schema'], +# ) +# transform = tfx.components.Transform( +# examples=example_gen.outputs['examples'], +# schema=schema_gen.outputs['schema'], +# module_file=taxi_module_file, +# ) +# trainer = tfx.components.Trainer( +# module_file=taxi_module_file, +# examples=transform.outputs['transformed_examples'], +# schema=schema_gen.outputs['schema'], +# transform_graph=transform.outputs['transform_graph'], +# train_args=tfx.proto.TrainArgs(num_steps=10), +# eval_args=tfx.proto.EvalArgs(num_steps=5), +# ) +# # Set the TFMA config for Model Evaluation and Validation. +# eval_config = tfma.EvalConfig( +# model_specs=[ +# tfma.ModelSpec( +# signature_name='serving_default', +# label_key='tips_xf', +# preprocessing_function_names=['transform_features']) +# ], +# metrics_specs=[ +# tfma.MetricsSpec( +# # The metrics added here are in addition to those saved with the +# # model (assuming either a keras model or EvalSavedModel is used). +# # Any metrics added into the saved model (for example using +# # model.compile(..., metrics=[...]), etc) will be computed +# # automatically. +# metrics=[tfma.MetricConfig(class_name='ExampleCount')], +# # To add validation thresholds for metrics saved with the model, +# # add them keyed by metric name to the thresholds map. +# thresholds={ +# 'binary_accuracy': +# tfma.MetricThreshold( +# value_threshold=tfma.GenericValueThreshold( +# lower_bound={'value': 0.5}), +# change_threshold=tfma.GenericChangeThreshold( +# direction=tfma.MetricDirection.HIGHER_IS_BETTER, +# absolute={'value': -1e-10})) +# }) +# ], +# slicing_specs=[ +# # An empty slice spec means the overall slice, i.e. the whole dataset. +# tfma.SlicingSpec(), +# # Data can be sliced along a feature column. In this case, data is +# # sliced along feature column trip_start_hour. +# tfma.SlicingSpec(feature_keys=['trip_start_hour']) +# ]) + +# evaluator = tfx.components.Evaluator( +# examples=example_gen.outputs['examples'], +# model=trainer.outputs['model'], +# eval_config=eval_config, +# ) + +# pusher = tfx.components.Pusher( +# model=trainer.outputs['model'], +# model_blessing=evaluator.outputs['blessing'], +# push_destination=push_destination, +# ) + +# return tfx.dsl.Pipeline( +# pipeline_name='parameterized_tfx_oss', +# pipeline_root=pipeline_root, +# components=[ +# example_gen, statistics_gen, schema_gen, example_validator, +# transform, trainer, evaluator, pusher +# ], +# enable_cache=enable_cache, +# ) + + +# if __name__ == '__main__': +# enable_cache = True +# pipeline = _create_pipeline( +# _pipeline_root, +# _data_root, +# _taxi_module_file_param, +# _push_destination, +# enable_cache=enable_cache, +# ) +# # Make sure the version of TFX image used is consistent with the version of +# # TFX SDK. +# config = tfx.orchestration.experimental.KubeflowDagRunnerConfig( +# kubeflow_metadata_config=tfx.orchestration.experimental +# .get_default_kubeflow_metadata_config(), +# tfx_image='gcr.io/tfx-oss-public/tfx:%s' % tfx.__version__, +# pipeline_operator_funcs=kubeflow_dag_runner +# .get_default_pipeline_operator_funcs(use_gcp_sa=False) + [ +# kfp.deprecated.onprem.add_default_resource_spec( +# memory_limit='2Gi', cpu_limit='2', cpu_request='1'), +# ], +# ) +# kfp_runner = tfx.orchestration.experimental.KubeflowDagRunner( +# output_filename=__file__ + '.yaml', +# config=config, +# ) + +# kfp_runner.run(pipeline) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py index 3293bbb5594..60385e13f06 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py @@ -12,29 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func +# TODO: TFX oss uses early depencies, uncomment until fix +# import json +# import kfp.deprecated as kfp +# from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func -bucket = 'kfp-ci' +# bucket = 'kfp-ci' -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'parameterized_tfx_oss.py'), - pipeline_file_compile_path=relative_path( - __file__, 'parameterized_tfx_oss.py.yaml'), - arguments={ - 'pipeline-root': - f'gs://{bucket}/tfx_taxi_simple', - 'push_destination': - json.dumps({ - "filesystem": { - "base_directory": - f"gs://{bucket}/tfx_taxi_simple/serving_model" - } - }) - }, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - timeout_mins=40, - ), -]) +# run_pipeline_func([ +# TestCase( +# pipeline_file=relative_path(__file__, 'parameterized_tfx_oss.py'), +# pipeline_file_compile_path=relative_path( +# __file__, 'parameterized_tfx_oss.py.yaml'), +# arguments={ +# 'pipeline-root': +# f'gs://{bucket}/tfx_taxi_simple', +# 'push_destination': +# json.dumps({ +# "filesystem": { +# "base_directory": +# f"gs://{bucket}/tfx_taxi_simple/serving_model" +# } +# }) +# }, +# mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, +# timeout_mins=40, +# ), +# ]) diff --git a/samples/test/config-integration.yaml b/samples/test/config-integration.yaml index 936e6aadd59..279be3c1e79 100644 --- a/samples/test/config-integration.yaml +++ b/samples/test/config-integration.yaml @@ -26,8 +26,9 @@ # Integration Samples - name: dataflow path: samples.core.dataflow.dataflow_test -- name: parameterized_tfx_oss - path: samples.core.parameterized_tfx_oss.parameterized_tfx_oss_test +# TODO: reenable when kfp dependency is updated +# - name: parameterized_tfx_oss +# path: samples.core.parameterized_tfx_oss.parameterized_tfx_oss_test # The following samples were in integration test, but we disabled them. # From c1f7746c72c0790f79b33533158b6d9679a199a4 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Tue, 22 Feb 2022 00:27:47 -0800 Subject: [PATCH 27/39] fix sampe --- samples/core/secret/secret.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/core/secret/secret.py b/samples/core/secret/secret.py index d1193516041..0145dc7e70b 100644 --- a/samples/core/secret/secret.py +++ b/samples/core/secret/secret.py @@ -14,11 +14,11 @@ # limitations under the License. -from kfp.deprecated import dsl, compiler +from kfp.deprecated import dsl, compiler, components # Accessing GCS using the Google Cloud SDK command-line programs -gcs_list_items_op = kfp.components.load_component_from_text(''' +gcs_list_items_op = components.load_component_from_text(''' name: GCS - List items inputs: - {name: Uri} @@ -49,7 +49,7 @@ def gcs_list_buckets(): print(bucket.name) -gcs_list_buckets_op = kfp.components.create_component_from_func( +gcs_list_buckets_op = components.create_component_from_func( gcs_list_buckets, base_image='python:3.7', packages_to_install=['google-cloud-storage==1.31.2'], From ad3e3c2ed933ca4e165050404e2c1f58eea256f2 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 2 Mar 2022 09:44:32 -0800 Subject: [PATCH 28/39] move client down --- samples/core/use_run_info/use_run_id.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/core/use_run_info/use_run_id.py b/samples/core/use_run_info/use_run_id.py index d1adc378e10..95bdc7f136a 100644 --- a/samples/core/use_run_info/use_run_id.py +++ b/samples/core/use_run_info/use_run_id.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.deprecated import dsl, compiler, Client, components +from kfp.deprecated import dsl, compiler, components def get_run_info(run_id: str): @@ -23,6 +23,7 @@ def get_run_info(run_id: str): # namespace, but for full Kubeflow deployment, you need to edit this to # http://ml-pipeline.kubeflow:8888, because your pipelines are running in # user namespaces, but the API is at kubeflow namespace. + from kfp.deprecated import Client client = Client(host='http://ml-pipeline:8888') run_info = client.get_run(run_id=run_id) # Hide verbose info From f05c81c9cd3fa8a0f1cedc32f5fb59a39e618775 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 2 Mar 2022 10:21:45 -0800 Subject: [PATCH 29/39] change to kfp --- samples/core/use_run_info/use_run_id.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/core/use_run_info/use_run_id.py b/samples/core/use_run_info/use_run_id.py index 95bdc7f136a..bd298c5f1e4 100644 --- a/samples/core/use_run_info/use_run_id.py +++ b/samples/core/use_run_info/use_run_id.py @@ -23,8 +23,8 @@ def get_run_info(run_id: str): # namespace, but for full Kubeflow deployment, you need to edit this to # http://ml-pipeline.kubeflow:8888, because your pipelines are running in # user namespaces, but the API is at kubeflow namespace. - from kfp.deprecated import Client - client = Client(host='http://ml-pipeline:8888') + import kfp + client = kfp.deprecated.Client(host='http://ml-pipeline:8888') run_info = client.get_run(run_id=run_id) # Hide verbose info print(run_info.run) From b033c2a7bea9b4455c502b4c00ded3ff2d95601d Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 2 Mar 2022 10:56:45 -0800 Subject: [PATCH 30/39] add import alias --- samples/core/train_until_good/train_until_good.py | 2 +- samples/core/use_run_info/use_run_id.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/core/train_until_good/train_until_good.py b/samples/core/train_until_good/train_until_good.py index 5948a2bd604..2a68579fa31 100644 --- a/samples/core/train_until_good/train_until_good.py +++ b/samples/core/train_until_good/train_until_good.py @@ -17,7 +17,7 @@ # The main pipeline trains the initial model and then gradually trains the model # some more until the model evaluation metrics are good enough. -from kfp import components, dsl, Client +from kfp.deprecated import components, dsl, Client chicago_taxi_dataset_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml') xgboost_train_on_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Train/component.yaml') diff --git a/samples/core/use_run_info/use_run_id.py b/samples/core/use_run_info/use_run_id.py index bd298c5f1e4..eb24309c0be 100644 --- a/samples/core/use_run_info/use_run_id.py +++ b/samples/core/use_run_info/use_run_id.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import kfp.deprecated as kfp from kfp.deprecated import dsl, compiler, components @@ -23,8 +24,7 @@ def get_run_info(run_id: str): # namespace, but for full Kubeflow deployment, you need to edit this to # http://ml-pipeline.kubeflow:8888, because your pipelines are running in # user namespaces, but the API is at kubeflow namespace. - import kfp - client = kfp.deprecated.Client(host='http://ml-pipeline:8888') + client = kfp.Client(host='http://ml-pipeline:8888') run_info = client.get_run(run_id=run_id) # Hide verbose info print(run_info.run) From 5f1abef351eb65645991e3bde3f214d80e5ef693 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 2 Mar 2022 12:51:21 -0800 Subject: [PATCH 31/39] fix --- samples/core/use_run_info/use_run_id.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/core/use_run_info/use_run_id.py b/samples/core/use_run_info/use_run_id.py index eb24309c0be..00f9cac9a07 100644 --- a/samples/core/use_run_info/use_run_id.py +++ b/samples/core/use_run_info/use_run_id.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.deprecated as kfp from kfp.deprecated import dsl, compiler, components @@ -24,6 +23,7 @@ def get_run_info(run_id: str): # namespace, but for full Kubeflow deployment, you need to edit this to # http://ml-pipeline.kubeflow:8888, because your pipelines are running in # user namespaces, but the API is at kubeflow namespace. + import kfp.deprecated as kfp client = kfp.Client(host='http://ml-pipeline:8888') run_info = client.get_run(run_id=run_id) # Hide verbose info From 7a9dcba350eb32afd7da46693b1e11b53ba9bf3f Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 2 Mar 2022 15:37:58 -0800 Subject: [PATCH 32/39] runid --- samples/core/use_run_info/use_run_id.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/samples/core/use_run_info/use_run_id.py b/samples/core/use_run_info/use_run_id.py index 00f9cac9a07..83dbad763af 100644 --- a/samples/core/use_run_info/use_run_id.py +++ b/samples/core/use_run_info/use_run_id.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.deprecated import dsl, compiler, components - +import kfp +import kfp.dsl as dsl def get_run_info(run_id: str): """Example of getting run info for current pipeline run.""" @@ -23,14 +23,13 @@ def get_run_info(run_id: str): # namespace, but for full Kubeflow deployment, you need to edit this to # http://ml-pipeline.kubeflow:8888, because your pipelines are running in # user namespaces, but the API is at kubeflow namespace. - import kfp.deprecated as kfp client = kfp.Client(host='http://ml-pipeline:8888') run_info = client.get_run(run_id=run_id) # Hide verbose info print(run_info.run) -get_run_info_component = components.create_component_from_func( +get_run_info_component = kfp.components.create_component_from_func( func=get_run_info, packages_to_install=['kfp'], ) @@ -40,11 +39,11 @@ def get_run_info(run_id: str): name='use-run-id', description='A pipeline that demonstrates how to use run information, including run ID etc.' ) -def pipeline_use_run_id(run_id: str = dsl.RUN_ID_PLACEHOLDER): +def pipeline_use_run_id(run_id: str = kfp.dsl.RUN_ID_PLACEHOLDER): """kfp.dsl.RUN_ID_PLACEHOLDER inside a pipeline parameter will be populated with KFP Run ID at runtime.""" run_info_op = get_run_info_component(run_id=run_id) if __name__ == '__main__': - compiler.Compiler().compile(pipeline_use_run_id, __file__ + '.yaml') + kfp.compiler.Compiler().compile(pipeline_use_run_id, __file__ + '.yaml') From e506d9ea3d336fe6fd62814b7e15db5028e73b1f Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 2 Mar 2022 21:44:39 -0800 Subject: [PATCH 33/39] fix dsl --- samples/core/use_run_info/use_run_id.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/core/use_run_info/use_run_id.py b/samples/core/use_run_info/use_run_id.py index 83dbad763af..4013f34dd6f 100644 --- a/samples/core/use_run_info/use_run_id.py +++ b/samples/core/use_run_info/use_run_id.py @@ -14,7 +14,6 @@ # limitations under the License. import kfp -import kfp.dsl as dsl def get_run_info(run_id: str): """Example of getting run info for current pipeline run.""" @@ -35,7 +34,7 @@ def get_run_info(run_id: str): ) -@dsl.pipeline( +@kfp.dsl.pipeline( name='use-run-id', description='A pipeline that demonstrates how to use run information, including run ID etc.' ) From 64218afbbcd2ece1e881308e943eaa6d985959db Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 2 Mar 2022 22:28:53 -0800 Subject: [PATCH 34/39] only use kfp for function --- samples/core/use_run_info/use_run_id.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/samples/core/use_run_info/use_run_id.py b/samples/core/use_run_info/use_run_id.py index 4013f34dd6f..9dbce233117 100644 --- a/samples/core/use_run_info/use_run_id.py +++ b/samples/core/use_run_info/use_run_id.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +import kfp.deprecated as kfp +from kfp.deprecated import components, dsl, compiler def get_run_info(run_id: str): """Example of getting run info for current pipeline run.""" @@ -22,19 +23,20 @@ def get_run_info(run_id: str): # namespace, but for full Kubeflow deployment, you need to edit this to # http://ml-pipeline.kubeflow:8888, because your pipelines are running in # user namespaces, but the API is at kubeflow namespace. + import kfp client = kfp.Client(host='http://ml-pipeline:8888') run_info = client.get_run(run_id=run_id) # Hide verbose info print(run_info.run) -get_run_info_component = kfp.components.create_component_from_func( +get_run_info_component = components.create_component_from_func( func=get_run_info, packages_to_install=['kfp'], ) -@kfp.dsl.pipeline( +@dsl.pipeline( name='use-run-id', description='A pipeline that demonstrates how to use run information, including run ID etc.' ) @@ -45,4 +47,4 @@ def pipeline_use_run_id(run_id: str = kfp.dsl.RUN_ID_PLACEHOLDER): if __name__ == '__main__': - kfp.compiler.Compiler().compile(pipeline_use_run_id, __file__ + '.yaml') + compiler.Compiler().compile(pipeline_use_run_id, __file__ + '.yaml') From b439b04b030fe585fcea9ebe48898aec1d49a02e Mon Sep 17 00:00:00 2001 From: Yaqi Date: Wed, 2 Mar 2022 23:00:58 -0800 Subject: [PATCH 35/39] revert train_until_good --- samples/core/train_until_good/train_until_good.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/core/train_until_good/train_until_good.py b/samples/core/train_until_good/train_until_good.py index 2a68579fa31..5948a2bd604 100644 --- a/samples/core/train_until_good/train_until_good.py +++ b/samples/core/train_until_good/train_until_good.py @@ -17,7 +17,7 @@ # The main pipeline trains the initial model and then gradually trains the model # some more until the model evaluation metrics are good enough. -from kfp.deprecated import components, dsl, Client +from kfp import components, dsl, Client chicago_taxi_dataset_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml') xgboost_train_on_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Train/component.yaml') From 5dde73e408fb3569c3c0c14abf8c0d5741c635ed Mon Sep 17 00:00:00 2001 From: Yaqi Date: Thu, 3 Mar 2022 12:19:32 -0800 Subject: [PATCH 36/39] tfx test --- .../parameterized_tfx_oss.py | 332 +++++++++--------- .../parameterized_tfx_oss_test.py | 49 ++- 2 files changed, 189 insertions(+), 192 deletions(-) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py index 7c0a77ec811..e14bfc4b141 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py @@ -13,170 +13,168 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: TFX oss uses early depencies, uncomment until fix - -# import json -# import os - -# import kfp.deprecated as kfp -# import kfp.deprecated.onprem -# import tfx.orchestration.kubeflow.kubeflow_dag_runner as kubeflow_dag_runner -# import tensorflow_model_analysis as tfma -# from tfx import v1 as tfx - -# # Define pipeline params used for pipeline execution. -# # Path to the module file, should be a GCS path, -# # or a module file baked in the docker image used by the pipeline. -# _taxi_module_file_param = tfx.dsl.experimental.RuntimeParameter( -# name='module-file', -# default='/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/taxi_utils_native_keras.py', -# ptype=str, -# ) - -# # Path to the CSV data file, under which their should be a data.csv file. -# _data_root = '/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/data/simple' - -# # Path of pipeline root, should be a GCS path. -# _pipeline_root = os.path.join('gs://{{kfp-default-bucket}}', 'tfx_taxi_simple', -# kfp.dsl.RUN_ID_PLACEHOLDER) - -# # Path that ML models are pushed, should be a GCS path. -# _serving_model_dir = os.path.join('gs://your-bucket', 'serving_model', -# 'tfx_taxi_simple') -# _push_destination = tfx.dsl.experimental.RuntimeParameter( -# name='push_destination', -# default=json.dumps({'filesystem': { -# 'base_directory': _serving_model_dir -# }}), -# ptype=str, -# ) - - -# def _create_pipeline( -# pipeline_root: str, -# csv_input_location: str, -# taxi_module_file: tfx.dsl.experimental.RuntimeParameter, -# push_destination: tfx.dsl.experimental.RuntimeParameter, -# enable_cache: bool, -# ): -# """Creates a simple Kubeflow-based Chicago Taxi TFX pipeline. - -# Args: -# pipeline_root: The root of the pipeline output. -# csv_input_location: The location of the input data directory. -# taxi_module_file: The location of the module file for Transform/Trainer. -# enable_cache: Whether to enable cache or not. - -# Returns: -# A logical TFX pipeline.Pipeline object. -# """ -# example_gen = tfx.components.CsvExampleGen(input_base=csv_input_location) -# statistics_gen = tfx.components.StatisticsGen( -# examples=example_gen.outputs['examples']) -# schema_gen = tfx.components.SchemaGen( -# statistics=statistics_gen.outputs['statistics'], -# infer_feature_shape=False, -# ) -# example_validator = tfx.components.ExampleValidator( -# statistics=statistics_gen.outputs['statistics'], -# schema=schema_gen.outputs['schema'], -# ) -# transform = tfx.components.Transform( -# examples=example_gen.outputs['examples'], -# schema=schema_gen.outputs['schema'], -# module_file=taxi_module_file, -# ) -# trainer = tfx.components.Trainer( -# module_file=taxi_module_file, -# examples=transform.outputs['transformed_examples'], -# schema=schema_gen.outputs['schema'], -# transform_graph=transform.outputs['transform_graph'], -# train_args=tfx.proto.TrainArgs(num_steps=10), -# eval_args=tfx.proto.EvalArgs(num_steps=5), -# ) -# # Set the TFMA config for Model Evaluation and Validation. -# eval_config = tfma.EvalConfig( -# model_specs=[ -# tfma.ModelSpec( -# signature_name='serving_default', -# label_key='tips_xf', -# preprocessing_function_names=['transform_features']) -# ], -# metrics_specs=[ -# tfma.MetricsSpec( -# # The metrics added here are in addition to those saved with the -# # model (assuming either a keras model or EvalSavedModel is used). -# # Any metrics added into the saved model (for example using -# # model.compile(..., metrics=[...]), etc) will be computed -# # automatically. -# metrics=[tfma.MetricConfig(class_name='ExampleCount')], -# # To add validation thresholds for metrics saved with the model, -# # add them keyed by metric name to the thresholds map. -# thresholds={ -# 'binary_accuracy': -# tfma.MetricThreshold( -# value_threshold=tfma.GenericValueThreshold( -# lower_bound={'value': 0.5}), -# change_threshold=tfma.GenericChangeThreshold( -# direction=tfma.MetricDirection.HIGHER_IS_BETTER, -# absolute={'value': -1e-10})) -# }) -# ], -# slicing_specs=[ -# # An empty slice spec means the overall slice, i.e. the whole dataset. -# tfma.SlicingSpec(), -# # Data can be sliced along a feature column. In this case, data is -# # sliced along feature column trip_start_hour. -# tfma.SlicingSpec(feature_keys=['trip_start_hour']) -# ]) - -# evaluator = tfx.components.Evaluator( -# examples=example_gen.outputs['examples'], -# model=trainer.outputs['model'], -# eval_config=eval_config, -# ) - -# pusher = tfx.components.Pusher( -# model=trainer.outputs['model'], -# model_blessing=evaluator.outputs['blessing'], -# push_destination=push_destination, -# ) - -# return tfx.dsl.Pipeline( -# pipeline_name='parameterized_tfx_oss', -# pipeline_root=pipeline_root, -# components=[ -# example_gen, statistics_gen, schema_gen, example_validator, -# transform, trainer, evaluator, pusher -# ], -# enable_cache=enable_cache, -# ) - - -# if __name__ == '__main__': -# enable_cache = True -# pipeline = _create_pipeline( -# _pipeline_root, -# _data_root, -# _taxi_module_file_param, -# _push_destination, -# enable_cache=enable_cache, -# ) -# # Make sure the version of TFX image used is consistent with the version of -# # TFX SDK. -# config = tfx.orchestration.experimental.KubeflowDagRunnerConfig( -# kubeflow_metadata_config=tfx.orchestration.experimental -# .get_default_kubeflow_metadata_config(), -# tfx_image='gcr.io/tfx-oss-public/tfx:%s' % tfx.__version__, -# pipeline_operator_funcs=kubeflow_dag_runner -# .get_default_pipeline_operator_funcs(use_gcp_sa=False) + [ -# kfp.deprecated.onprem.add_default_resource_spec( -# memory_limit='2Gi', cpu_limit='2', cpu_request='1'), -# ], -# ) -# kfp_runner = tfx.orchestration.experimental.KubeflowDagRunner( -# output_filename=__file__ + '.yaml', -# config=config, -# ) - -# kfp_runner.run(pipeline) +import json +import os + +import kfp.deprecated as kfp +import kfp.deprecated.onprem +import tfx.orchestration.kubeflow.kubeflow_dag_runner as kubeflow_dag_runner +import tensorflow_model_analysis as tfma +from tfx import v1 as tfx + +# Define pipeline params used for pipeline execution. +# Path to the module file, should be a GCS path, +# or a module file baked in the docker image used by the pipeline. +_taxi_module_file_param = tfx.dsl.experimental.RuntimeParameter( + name='module-file', + default='/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/taxi_utils_native_keras.py', + ptype=str, +) + +# Path to the CSV data file, under which their should be a data.csv file. +_data_root = '/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/data/simple' + +# Path of pipeline root, should be a GCS path. +_pipeline_root = os.path.join('gs://{{kfp-default-bucket}}', 'tfx_taxi_simple', + kfp.dsl.RUN_ID_PLACEHOLDER) + +# Path that ML models are pushed, should be a GCS path. +_serving_model_dir = os.path.join('gs://your-bucket', 'serving_model', + 'tfx_taxi_simple') +_push_destination = tfx.dsl.experimental.RuntimeParameter( + name='push_destination', + default=json.dumps({'filesystem': { + 'base_directory': _serving_model_dir + }}), + ptype=str, +) + + +def _create_pipeline( + pipeline_root: str, + csv_input_location: str, + taxi_module_file: tfx.dsl.experimental.RuntimeParameter, + push_destination: tfx.dsl.experimental.RuntimeParameter, + enable_cache: bool, +): + """Creates a simple Kubeflow-based Chicago Taxi TFX pipeline. + + Args: + pipeline_root: The root of the pipeline output. + csv_input_location: The location of the input data directory. + taxi_module_file: The location of the module file for Transform/Trainer. + enable_cache: Whether to enable cache or not. + + Returns: + A logical TFX pipeline.Pipeline object. + """ + example_gen = tfx.components.CsvExampleGen(input_base=csv_input_location) + statistics_gen = tfx.components.StatisticsGen( + examples=example_gen.outputs['examples']) + schema_gen = tfx.components.SchemaGen( + statistics=statistics_gen.outputs['statistics'], + infer_feature_shape=False, + ) + example_validator = tfx.components.ExampleValidator( + statistics=statistics_gen.outputs['statistics'], + schema=schema_gen.outputs['schema'], + ) + transform = tfx.components.Transform( + examples=example_gen.outputs['examples'], + schema=schema_gen.outputs['schema'], + module_file=taxi_module_file, + ) + trainer = tfx.components.Trainer( + module_file=taxi_module_file, + examples=transform.outputs['transformed_examples'], + schema=schema_gen.outputs['schema'], + transform_graph=transform.outputs['transform_graph'], + train_args=tfx.proto.TrainArgs(num_steps=10), + eval_args=tfx.proto.EvalArgs(num_steps=5), + ) + # Set the TFMA config for Model Evaluation and Validation. + eval_config = tfma.EvalConfig( + model_specs=[ + tfma.ModelSpec( + signature_name='serving_default', + label_key='tips_xf', + preprocessing_function_names=['transform_features']) + ], + metrics_specs=[ + tfma.MetricsSpec( + # The metrics added here are in addition to those saved with the + # model (assuming either a keras model or EvalSavedModel is used). + # Any metrics added into the saved model (for example using + # model.compile(..., metrics=[...]), etc) will be computed + # automatically. + metrics=[tfma.MetricConfig(class_name='ExampleCount')], + # To add validation thresholds for metrics saved with the model, + # add them keyed by metric name to the thresholds map. + thresholds={ + 'binary_accuracy': + tfma.MetricThreshold( + value_threshold=tfma.GenericValueThreshold( + lower_bound={'value': 0.5}), + change_threshold=tfma.GenericChangeThreshold( + direction=tfma.MetricDirection.HIGHER_IS_BETTER, + absolute={'value': -1e-10})) + }) + ], + slicing_specs=[ + # An empty slice spec means the overall slice, i.e. the whole dataset. + tfma.SlicingSpec(), + # Data can be sliced along a feature column. In this case, data is + # sliced along feature column trip_start_hour. + tfma.SlicingSpec(feature_keys=['trip_start_hour']) + ]) + + evaluator = tfx.components.Evaluator( + examples=example_gen.outputs['examples'], + model=trainer.outputs['model'], + eval_config=eval_config, + ) + + pusher = tfx.components.Pusher( + model=trainer.outputs['model'], + model_blessing=evaluator.outputs['blessing'], + push_destination=push_destination, + ) + + return tfx.dsl.Pipeline( + pipeline_name='parameterized_tfx_oss', + pipeline_root=pipeline_root, + components=[ + example_gen, statistics_gen, schema_gen, example_validator, + transform, trainer, evaluator, pusher + ], + enable_cache=enable_cache, + ) + + +if __name__ == '__main__': + enable_cache = True + pipeline = _create_pipeline( + _pipeline_root, + _data_root, + _taxi_module_file_param, + _push_destination, + enable_cache=enable_cache, + ) + # Make sure the version of TFX image used is consistent with the version of + # TFX SDK. + config = tfx.orchestration.experimental.KubeflowDagRunnerConfig( + kubeflow_metadata_config=tfx.orchestration.experimental + .get_default_kubeflow_metadata_config(), + tfx_image='gcr.io/tfx-oss-public/tfx:%s' % tfx.__version__, + pipeline_operator_funcs=kubeflow_dag_runner + .get_default_pipeline_operator_funcs(use_gcp_sa=False) + [ + kfp.deprecated.onprem.add_default_resource_spec( + memory_limit='2Gi', cpu_limit='2', cpu_request='1'), + ], + ) + kfp_runner = tfx.orchestration.experimental.KubeflowDagRunner( + output_filename=__file__ + '.yaml', + config=config, + ) + + kfp_runner.run(pipeline) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py index 60385e13f06..3293bbb5594 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py @@ -12,30 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: TFX oss uses early depencies, uncomment until fix -# import json -# import kfp.deprecated as kfp -# from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func +import json +import kfp.deprecated as kfp +from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func -# bucket = 'kfp-ci' +bucket = 'kfp-ci' -# run_pipeline_func([ -# TestCase( -# pipeline_file=relative_path(__file__, 'parameterized_tfx_oss.py'), -# pipeline_file_compile_path=relative_path( -# __file__, 'parameterized_tfx_oss.py.yaml'), -# arguments={ -# 'pipeline-root': -# f'gs://{bucket}/tfx_taxi_simple', -# 'push_destination': -# json.dumps({ -# "filesystem": { -# "base_directory": -# f"gs://{bucket}/tfx_taxi_simple/serving_model" -# } -# }) -# }, -# mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, -# timeout_mins=40, -# ), -# ]) +run_pipeline_func([ + TestCase( + pipeline_file=relative_path(__file__, 'parameterized_tfx_oss.py'), + pipeline_file_compile_path=relative_path( + __file__, 'parameterized_tfx_oss.py.yaml'), + arguments={ + 'pipeline-root': + f'gs://{bucket}/tfx_taxi_simple', + 'push_destination': + json.dumps({ + "filesystem": { + "base_directory": + f"gs://{bucket}/tfx_taxi_simple/serving_model" + } + }) + }, + mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, + timeout_mins=40, + ), +]) From d5adb28d412639ca921c1b314a88d497085e92c7 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Thu, 3 Mar 2022 13:51:27 -0800 Subject: [PATCH 37/39] kfp --- samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py index e14bfc4b141..183ad788ddd 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py @@ -16,8 +16,8 @@ import json import os -import kfp.deprecated as kfp -import kfp.deprecated.onprem +import kfp +from kfp.deprecated import onprem import tfx.orchestration.kubeflow.kubeflow_dag_runner as kubeflow_dag_runner import tensorflow_model_analysis as tfma from tfx import v1 as tfx @@ -168,7 +168,7 @@ def _create_pipeline( tfx_image='gcr.io/tfx-oss-public/tfx:%s' % tfx.__version__, pipeline_operator_funcs=kubeflow_dag_runner .get_default_pipeline_operator_funcs(use_gcp_sa=False) + [ - kfp.deprecated.onprem.add_default_resource_spec( + onprem.add_default_resource_spec( memory_limit='2Gi', cpu_limit='2', cpu_request='1'), ], ) From 4abe9413b67a447614810b19af1138b5a8286681 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Thu, 3 Mar 2022 15:19:47 -0800 Subject: [PATCH 38/39] try import --- .../core/parameterized_tfx_oss/parameterized_tfx_oss.py | 9 ++++++--- .../parameterized_tfx_oss/parameterized_tfx_oss_test.py | 7 ++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py index 183ad788ddd..945312d88bf 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py @@ -16,8 +16,11 @@ import json import os -import kfp -from kfp.deprecated import onprem +try: + import kfp.deprecated as kfp +except ImportError: + import kfp + import tfx.orchestration.kubeflow.kubeflow_dag_runner as kubeflow_dag_runner import tensorflow_model_analysis as tfma from tfx import v1 as tfx @@ -168,7 +171,7 @@ def _create_pipeline( tfx_image='gcr.io/tfx-oss-public/tfx:%s' % tfx.__version__, pipeline_operator_funcs=kubeflow_dag_runner .get_default_pipeline_operator_funcs(use_gcp_sa=False) + [ - onprem.add_default_resource_spec( + kfp.onprem.add_default_resource_spec( memory_limit='2Gi', cpu_limit='2', cpu_request='1'), ], ) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py index 3293bbb5594..225e5a08494 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py @@ -13,7 +13,12 @@ # limitations under the License. import json -import kfp.deprecated as kfp + +try: + import kfp.deprecated as kfp +except ImportError: + import kfp + from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func bucket = 'kfp-ci' From e96fc175009526bdfc79bb10ac5036c2b15f5201 Mon Sep 17 00:00:00 2001 From: Yaqi Date: Thu, 3 Mar 2022 15:55:52 -0800 Subject: [PATCH 39/39] onprem --- samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py index 945312d88bf..0e1e0dc3967 100644 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py +++ b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py @@ -18,8 +18,10 @@ try: import kfp.deprecated as kfp + from kfp.deprecated import onprem except ImportError: import kfp + from kfp import onprem import tfx.orchestration.kubeflow.kubeflow_dag_runner as kubeflow_dag_runner import tensorflow_model_analysis as tfma @@ -171,7 +173,7 @@ def _create_pipeline( tfx_image='gcr.io/tfx-oss-public/tfx:%s' % tfx.__version__, pipeline_operator_funcs=kubeflow_dag_runner .get_default_pipeline_operator_funcs(use_gcp_sa=False) + [ - kfp.onprem.add_default_resource_spec( + onprem.add_default_resource_spec( memory_limit='2Gi', cpu_limit='2', cpu_request='1'), ], )