Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SDK - Moved python op pipeline compilation test to bridge tests #3323

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
17 changes: 5 additions & 12 deletions sdk/python/tests/components/test_python_op.py
Expand Up @@ -840,7 +840,7 @@ def test_packages_to_install_feature(self):
self.helper_test_component_using_local_call(task_factory2, arguments={}, expected_output_values={})


def test_end_to_end_python_component_pipeline_compilation(self):
def test_end_to_end_python_component_pipeline(self):
import kfp.components as comp

#Defining the Python function
Expand All @@ -858,25 +858,18 @@ def add(a: float, b: float) -> float:
add_op2 = comp.load_component_from_file(add_component_file)

#Building the pipeline
import kfp.dsl as dsl
@dsl.pipeline(
name='Calculation pipeline',
description='A pipeline that performs arithmetic calculations.'
)
def calc_pipeline(
a1,
a2='7',
a3='17',
):
task_1 = add_op(a1, a2)
task_2 = add_op2(a1, a2)
task_3 = add_op(task_1.output, task_2.output)
task_4 = add_op2(task_3.output, a3)
task_3 = add_op(task_1.outputs['Output'], task_2.outputs['Output'])
task_4 = add_op2(task_3.outputs['Output'], a3)

#Compiling the pipleine:
pipeline_filename = str(Path(temp_dir_name).joinpath(calc_pipeline.__name__ + '.pipeline.tar.gz'))
import kfp.compiler as compiler
compiler.Compiler().compile(calc_pipeline, pipeline_filename)
#Instantiating the pipleine:
calc_pipeline(42)


if __name__ == '__main__':
Expand Down
39 changes: 38 additions & 1 deletion sdk/python/tests/dsl/component_bridge_tests.py
Expand Up @@ -12,10 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.


import tempfile
import textwrap
import unittest
import kfp
from pathlib import Path
from kfp.components import load_component_from_text
from kfp.dsl.types import InconsistentTypeException

Expand Down Expand Up @@ -144,3 +145,39 @@ def test_type_compatibility_check_not_failing_when_type_is_ignored(self):
task_factory_b = load_component_from_text(component_b)
a_task = task_factory_a()
b_task = task_factory_b(in1=a_task.outputs['out1'].ignore_type())

def test_end_to_end_python_component_pipeline_compilation(self):
import kfp.components as comp

#Defining the Python function
def add(a: float, b: float) -> float:
'''Returns sum of two arguments'''
return a + b

with tempfile.TemporaryDirectory() as temp_dir_name:
add_component_file = str(Path(temp_dir_name).joinpath('add.component.yaml'))

#Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
add_op = comp.func_to_container_op(add, base_image='python:3.5', output_component_file=add_component_file)

#Checking that the component artifact is usable:
add_op2 = comp.load_component_from_file(add_component_file)

#Building the pipeline
@kfp.dsl.pipeline(
name='Calculation pipeline',
description='A pipeline that performs arithmetic calculations.'
)
def calc_pipeline(
a1,
a2='7',
a3='17',
):
task_1 = add_op(a1, a2)
task_2 = add_op2(a1, a2)
task_3 = add_op(task_1.output, task_2.output)
task_4 = add_op2(task_3.output, a3)

#Compiling the pipleine:
pipeline_filename = str(Path(temp_dir_name).joinpath(calc_pipeline.__name__ + '.pipeline.tar.gz'))
kfp.compiler.Compiler().compile(calc_pipeline, pipeline_filename)