diff --git a/docs/source/user_guide/pipeline/examples.rst b/docs/source/user_guide/pipeline/examples.rst index 87e0d5516..688e679b0 100644 --- a/docs/source/user_guide/pipeline/examples.rst +++ b/docs/source/user_guide/pipeline/examples.rst @@ -6,87 +6,6 @@ Create a pipeline .. tabs:: - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - - pipeline_step = PipelineStep( - name="Python_Script_Step", - description="A step running a python script", - infrastructure=infrastructure, - runtime=runtime - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step], - ) - - pipeline.create() - - - .. code-tab:: Python3 - :caption: Python (Alternative) - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = ( - CustomScriptStep() - .with_block_storage_size(200) - .with_shape_name("VM.Standard3.Flex") - .with_shape_config_details(ocpus=4, memory_in_gbs=32) - ) - - runtime = ( - ScriptRuntime() - .with_source("script.py") - .with_service_conda("generalml_p37_cpu_v1") - ) - - pipeline_step = ( - PipelineStep("Python_Script_Step") - .with_description("A step running a python script") - .with_infrastructure(infrastructure) - .with_runtime(runtime) - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = ( - Pipeline("A single step pipeline") - .with_compartment_id(compartment_id) - .with_project_id(project_id) - .with_step_details([pipeline_step]) - ) - - pipeline.create() - .. code-tab:: Python3 :caption: YAML @@ -132,73 +51,19 @@ Create a pipeline pipeline = Pipeline.from_yaml(yaml_string) pipeline.create() - - -Run a job as a step -=================== - -.. tabs:: + .. code-tab:: Python3 :caption: Python - from ads.jobs import Job, DataScienceJob, ScriptRuntime - from ads.pipeline import PipelineStep, Pipeline - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = DataScienceJob( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - - job = Job( - infrastructure=infrastructure, - runtime=runtime - ) - job.create() # create a job - - pipeline_step = PipelineStep( - name="Job_Step", - description="A step running a job", - job_id=job.id - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step], - ) - - pipeline.create() - - pipeline_run = pipeline.run() - - - .. code-tab:: Python3 - :caption: Python (Alternative) - - from ads.jobs import Job, DataScienceJob, ScriptRuntime - from ads.pipeline import Pipeline, PipelineStep + from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os with open("script.py", "w") as f: f.write("print('Hello World!')") infrastructure = ( - DataScienceJob() + CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32) @@ -210,18 +75,12 @@ Run a job as a step .with_service_conda("generalml_p37_cpu_v1") ) - job = ( - Job() + pipeline_step = ( + PipelineStep("Python_Script_Step") + .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(runtime) ) - job.create() # create a job - - pipeline_step = ( - PipelineStep("Job_Step") - .with_description("A step running a job") - .with_job_id(job.id) - ) compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"] @@ -235,7 +94,13 @@ Run a job as a step pipeline.create() - pipeline_run = pipeline.run() + + + +Run a job as a step +=================== + +.. tabs:: .. code-tab:: Python3 :caption: YAML @@ -292,64 +157,18 @@ Run a job as a step pipeline_run = pipeline.run() -Run a python script as a step -============================= - -.. tabs:: - .. code-tab:: Python3 :caption: Python - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - - pipeline_step = PipelineStep( - name="Python_Script_Step", - description="A step running a python script", - infrastructure=infrastructure, - runtime=runtime - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step], - ) - - pipeline.create() - - pipeline_run = pipeline.run() - - - .. code-tab:: Python3 - :caption: Python (Alternative) - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime + from ads.jobs import Job, DataScienceJob, ScriptRuntime + from ads.pipeline import Pipeline, PipelineStep import os with open("script.py", "w") as f: f.write("print('Hello World!')") infrastructure = ( - CustomScriptStep() + DataScienceJob() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32) @@ -361,12 +180,18 @@ Run a python script as a step .with_service_conda("generalml_p37_cpu_v1") ) - pipeline_step = ( - PipelineStep("Python_Script_Step") - .with_description("A step running a python script") + job = ( + Job() .with_infrastructure(infrastructure) .with_runtime(runtime) ) + job.create() # create a job + + pipeline_step = ( + PipelineStep("Job_Step") + .with_description("A step running a job") + .with_job_id(job.id) + ) compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"] @@ -382,6 +207,13 @@ Run a python script as a step pipeline_run = pipeline.run() + + +Run a python script as a step +============================= + +.. tabs:: + .. code-tab:: Python3 :caption: YAML @@ -430,59 +262,16 @@ Run a python script as a step pipeline_run = pipeline.run() - -Run a notebook as a step -======================== - -.. tabs:: .. code-tab:: Python3 :caption: Python - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, NotebookRuntime - import os - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - runtime = NotebookRuntime( - notebook_path_uri="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"}, - output_uri="oci://@/", - env={"GREETINGS": "Welcome to OCI Data Science"} - ) - - pipeline_step = PipelineStep( - name="Notebook_Step", - description="A step running a notebook", - infrastructure=infrastructure, - runtime=runtime - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step], - ) - - pipeline.create() - - pipeline_run = pipeline.run() - - - .. code-tab:: Python3 - :caption: Python (Alternative) - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, NotebookRuntime + from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os + with open("script.py", "w") as f: + f.write("print('Hello World!')") + infrastructure = ( CustomScriptStep() .with_block_storage_size(200) @@ -491,19 +280,14 @@ Run a notebook as a step ) runtime = ( - NotebookRuntime() - .with_notebook( - path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - encoding='utf-8' - ) - .with_service_conda("tensorflow26_p37_cpu_v2") - .with_environment_variable(GREETINGS="Welcome to OCI Data Science") - .with_output("oci://@/") + ScriptRuntime() + .with_source("script.py") + .with_service_conda("generalml_p37_cpu_v1") ) pipeline_step = ( - PipelineStep("Notebook_Step") - .with_description("A step running a notebook") + PipelineStep("Python_Script_Step") + .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(runtime) ) @@ -512,16 +296,23 @@ Run a notebook as a step project_id = os.environ["PROJECT_OCID"] pipeline = ( - Pipeline("A single step pipeline") - .with_compartment_id(compartment_id) - .with_project_id(project_id) - .with_step_details([pipeline_step]) - ) + Pipeline("A single step pipeline") + .with_compartment_id(compartment_id) + .with_project_id(project_id) + .with_step_details([pipeline_step]) + ) pipeline.create() pipeline_run = pipeline.run() + + +Run a notebook as a step +======================== + +.. tabs:: + .. code-tab:: Python3 :caption: YAML @@ -571,79 +362,13 @@ Run a notebook as a step pipeline.create() pipeline_run = pipeline.run() - - - -Run two steps with the same infrastructure -========================================== - -.. tabs:: - + .. code-tab:: Python3 :caption: Python - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - step_one_runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "generalml_p37_cpu_v1"} - ) - - pipeline_step_one = PipelineStep( - name="Python_Script_Step", - description="A step running a python script", - infrastructure=infrastructure, - runtime=step_one_runtime - ) - - step_two_runtime = NotebookRuntime( - notebook_path_uri="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"}, - output_uri="oci://@/", - env={"GREETINGS": "Welcome to OCI Data Science"} - ) - - pipeline_step_two = PipelineStep( - name="Notebook_Step", - description="A step running a notebook", - infrastructure=infrastructure, - runtime=step_two_runtime - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step_one, pipeline_step_two], - ) - - pipeline.create() - - pipeline_run = pipeline.run() - - - .. code-tab:: Python3 - :caption: Python (Alternative) - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime + from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, NotebookRuntime import os - with open("script.py", "w") as f: - f.write("print('Hello World!')") - infrastructure = ( CustomScriptStep() .with_block_storage_size(200) @@ -651,20 +376,7 @@ Run two steps with the same infrastructure .with_shape_config_details(ocpus=4, memory_in_gbs=32) ) - step_one_runtime = ( - ScriptRuntime() - .with_source("script.py") - .with_service_conda("generalml_p37_cpu_v1") - ) - - pipeline_step_one = ( - PipelineStep("Python_Script_Step") - .with_description("A step running a python script") - .with_infrastructure(infrastructure) - .with_runtime(step_one_runtime) - ) - - step_two_runtime = ( + runtime = ( NotebookRuntime() .with_notebook( path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", @@ -675,27 +387,34 @@ Run two steps with the same infrastructure .with_output("oci://@/") ) - pipeline_step_two = ( + pipeline_step = ( PipelineStep("Notebook_Step") .with_description("A step running a notebook") .with_infrastructure(infrastructure) - .with_runtime(step_two_runtime) + .with_runtime(runtime) ) compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"] pipeline = ( - Pipeline("A single step pipeline") - .with_compartment_id(compartment_id) - .with_project_id(project_id) - .with_step_details([pipeline_step_one, pipeline_step_two]) - ) + Pipeline("A single step pipeline") + .with_compartment_id(compartment_id) + .with_project_id(project_id) + .with_step_details([pipeline_step]) + ) pipeline.create() pipeline_run = pipeline.run() + + +Run two steps with the same infrastructure +========================================== + +.. tabs:: + .. code-tab:: Python3 :caption: YAML @@ -751,83 +470,19 @@ Run two steps with the same infrastructure kind: runtime spec: conda: - slug: tensorflow26_p37_cpu_v2 - type: service - env: - - name: GREETINGS - value: Welcome to OCI Data Science - notebookEncoding: utf-8 - notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb - outputURI: oci://@/ - type: notebook - type: pipeline - """.format(compartment_id=compartment_id, project_id=project_id) - - pipeline = Pipeline.from_yaml(yaml_string) - - pipeline.create() - - pipeline_run = pipeline.run() - - - - -Run two steps in parallel -========================= - -In the example below, when DAG is not specified, the steps in the pipeline run in parallel. - -.. tabs:: - - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - step_one_runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "generalml_p37_cpu_v1"} - ) - - pipeline_step_one = PipelineStep( - name="Python_Script_Step", - description="A step running a python script", - infrastructure=infrastructure, - runtime=step_one_runtime - ) - - step_two_runtime = NotebookRuntime( - notebook_path_uri="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"}, - output_uri="oci://@/", - env={"GREETINGS": "Welcome to OCI Data Science"} - ) - - pipeline_step_two = PipelineStep( - name="Notebook_Step", - description="A step running a notebook", - infrastructure=infrastructure, - runtime=step_two_runtime - ) - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] + slug: tensorflow26_p37_cpu_v2 + type: service + env: + - name: GREETINGS + value: Welcome to OCI Data Science + notebookEncoding: utf-8 + notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb + outputURI: oci://@/ + type: notebook + type: pipeline + """.format(compartment_id=compartment_id, project_id=project_id) - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step_one, pipeline_step_two], - ) + pipeline = Pipeline.from_yaml(yaml_string) pipeline.create() @@ -835,7 +490,7 @@ In the example below, when DAG is not specified, the steps in the pipeline run i .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime import os @@ -895,6 +550,16 @@ In the example below, when DAG is not specified, the steps in the pipeline run i pipeline_run = pipeline.run() + + + +Run two steps in parallel +========================= + +In the example below, when DAG is not specified, the steps in the pipeline run in parallel. + +.. tabs:: + .. code-tab:: Python3 :caption: YAML @@ -967,73 +632,11 @@ In the example below, when DAG is not specified, the steps in the pipeline run i pipeline.create() pipeline_run = pipeline.run() - - - -Run two steps sequentially -========================== - -.. tabs:: - + + .. code-tab:: Python3 :caption: Python - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - step_one_runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "generalml_p37_cpu_v1"} - ) - - pipeline_step_one = PipelineStep( - name="Python_Script_Step", - description="A step running a python script", - infrastructure=infrastructure, - runtime=step_one_runtime - ) - - step_two_runtime = NotebookRuntime( - notebook_path_uri="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"}, - output_uri="oci://@/", - env={"GREETINGS": "Welcome to OCI Data Science"} - ) - - pipeline_step_two = PipelineStep( - name="Notebook_Step", - description="A step running a notebook", - infrastructure=infrastructure, - runtime=step_two_runtime - ) - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step_one, pipeline_step_two], - dag=["Python_Script_Step >> Notebook_Step"], - ) - - pipeline.create() - - pipeline_run = pipeline.run() - - - .. code-tab:: Python3 - :caption: Python (Alternative) - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime import os @@ -1086,13 +689,20 @@ Run two steps sequentially .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step_one, pipeline_step_two]) - .with_dag(["Python_Script_Step >> Notebook_Step"]) ) pipeline.create() pipeline_run = pipeline.run() + + + +Run two steps sequentially +========================== + +.. tabs:: + .. code-tab:: Python3 :caption: YAML @@ -1168,83 +778,9 @@ Run two steps sequentially pipeline_run = pipeline.run() - -Run multiple steps with dependencies specified in DAG -===================================================== - -In this example, ``step_1`` and ``step_2`` run in parallel and ``step_3`` runs after ``step_1`` and ``step_2`` are complete. - -.. tabs:: - .. code-tab:: Python3 :caption: Python - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - script_runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - - notebook_runtime = NotebookRuntime( - notebook_path_uri="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - - pipeline_step_1 = PipelineStep( - name="step_1", - description="A step running a python script", - infrastructure=infrastructure, - runtime=script_runtime - ) - - pipeline_step_2 = PipelineStep( - name="step_2", - description="A step running a notebook", - infrastructure=infrastructure, - runtime=notebook_runtime - ) - - pipeline_step_3 = PipelineStep( - name="step_3", - description="A step running a python script", - infrastructure=infrastructure, - runtime=script_runtime - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="An example pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step_1, pipeline_step_2, pipeline_step_3], - dag=["(step_1, step_2) >> step_3"], - ) - - pipeline.create() # create the pipeline - pipeline.show() # visualize the pipeline - - pipeline_run = pipeline.run() # run the pipeline - - pipeline_run.show(wait=True) # watch the pipeline run status - - - - .. code-tab:: Python3 - :caption: Python (Alternative) - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime import os @@ -1258,61 +794,62 @@ In this example, ``step_1`` and ``step_2`` run in parallel and ``step_3`` runs a .with_shape_config_details(ocpus=4, memory_in_gbs=32) ) - script_runtime = ( + step_one_runtime = ( ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1") ) - notebook_runtime = ( + pipeline_step_one = ( + PipelineStep("Python_Script_Step") + .with_description("A step running a python script") + .with_infrastructure(infrastructure) + .with_runtime(step_one_runtime) + ) + + step_two_runtime = ( NotebookRuntime() .with_notebook( path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", encoding='utf-8' ) .with_service_conda("tensorflow26_p37_cpu_v2") + .with_environment_variable(GREETINGS="Welcome to OCI Data Science") + .with_output("oci://@/") ) - pipeline_step_1 = ( - PipelineStep("step_1") - .with_description("A step running a python script") - .with_infrastructure(infrastructure) - .with_runtime(script_runtime) - ) - - pipeline_step_2 = ( - PipelineStep("step_2") + pipeline_step_two = ( + PipelineStep("Notebook_Step") .with_description("A step running a notebook") .with_infrastructure(infrastructure) - .with_runtime(notebook_runtime) - ) - - pipeline_step_3 = ( - PipelineStep("step_3") - .with_description("A step running a python script") - .with_infrastructure(infrastructure) - .with_runtime(script_runtime) + .with_runtime(step_two_runtime) ) compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"] pipeline = ( - Pipeline("An example pipeline") + Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) - .with_step_details([pipeline_step_1, pipeline_step_2, pipeline_step_3]) - .with_dag(["(step_1, step_2) >> step_3"]) + .with_step_details([pipeline_step_one, pipeline_step_two]) + .with_dag(["Python_Script_Step >> Notebook_Step"]) ) - pipeline.create() # create the pipeline - pipeline.show() # visualize the pipeline + pipeline.create() - pipeline_run = pipeline.run() # run the pipeline + pipeline_run = pipeline.run() - pipeline_run.show(wait=True) # watch the pipeline run status + +Run multiple steps with dependencies specified in DAG +===================================================== + +In this example, ``step_1`` and ``step_2`` run in parallel and ``step_3`` runs after ``step_1`` and ``step_2`` are complete. + +.. tabs:: + .. code-tab:: Python3 :caption: YAML @@ -1408,46 +945,128 @@ In this example, ``step_1`` and ``step_2`` run in parallel and ``step_3`` runs a pipeline_run.show(wait=True) # watch the pipeline run status -Set environment variables in a step -=================================== - -.. tabs:: - .. code-tab:: Python3 :caption: Python - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, NotebookRuntime - import os + from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime + import os + + with open("script.py", "w") as f: + f.write("print('Hello World!')") + + infrastructure = ( + CustomScriptStep() + .with_block_storage_size(200) + .with_shape_name("VM.Standard3.Flex") + .with_shape_config_details(ocpus=4, memory_in_gbs=32) + ) + + script_runtime = ( + ScriptRuntime() + .with_source("script.py") + .with_service_conda("generalml_p37_cpu_v1") + ) + + notebook_runtime = ( + NotebookRuntime() + .with_notebook( + path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", + encoding='utf-8' + ) + .with_service_conda("tensorflow26_p37_cpu_v2") + ) - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, + pipeline_step_1 = ( + PipelineStep("step_1") + .with_description("A step running a python script") + .with_infrastructure(infrastructure) + .with_runtime(script_runtime) ) - runtime = NotebookRuntime( - notebook_path_uri="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"}, - output_uri="oci://@/", - env={"GREETINGS": "Welcome to OCI Data Science"} + pipeline_step_2 = ( + PipelineStep("step_2") + .with_description("A step running a notebook") + .with_infrastructure(infrastructure) + .with_runtime(notebook_runtime) ) - pipeline_step = PipelineStep( - name="Notebook_Step", - description="A step running a notebook", - infrastructure=infrastructure, - runtime=runtime + pipeline_step_3 = ( + PipelineStep("step_3") + .with_description("A step running a python script") + .with_infrastructure(infrastructure) + .with_runtime(script_runtime) ) compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"] - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step], - ) + pipeline = ( + Pipeline("An example pipeline") + .with_compartment_id(compartment_id) + .with_project_id(project_id) + .with_step_details([pipeline_step_1, pipeline_step_2, pipeline_step_3]) + .with_dag(["(step_1, step_2) >> step_3"]) + ) + + pipeline.create() # create the pipeline + pipeline.show() # visualize the pipeline + + pipeline_run = pipeline.run() # run the pipeline + + pipeline_run.show(wait=True) # watch the pipeline run status + + + +Set environment variables in a step +=================================== + +.. tabs:: + + .. code-tab:: Python3 + :caption: YAML + + from ads.pipeline import Pipeline + import os + + compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] + project_id = os.environ["PROJECT_OCID"] + + yaml_string = """ + kind: pipeline + spec: + compartmentId: {compartment_id} + displayName: A single step pipeline + projectId: {project_id} + stepDetails: + - kind: customScript + spec: + description: A step running a notebook + infrastructure: + kind: infrastructure + spec: + blockStorageSize: 200 + shapeConfigDetails: + memoryInGBs: 32 + ocpus: 4 + shapeName: VM.Standard3.Flex + name: Notebook_Step + runtime: + kind: runtime + spec: + conda: + slug: tensorflow26_p37_cpu_v2 + type: service + env: + - name: GREETINGS + value: Welcome to OCI Data Science + notebookEncoding: utf-8 + notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb + outputURI: oci://@/ + type: notebook + type: pipeline + """.format(compartment_id=compartment_id, project_id=project_id) + + pipeline = Pipeline.from_yaml(yaml_string) pipeline.create() @@ -1455,7 +1074,7 @@ Set environment variables in a step .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, NotebookRuntime import os @@ -1499,6 +1118,14 @@ Set environment variables in a step pipeline_run = pipeline.run() + + + +Watch status update on a pipeline run +===================================== + +.. tabs:: + .. code-tab:: Python3 :caption: YAML @@ -1508,6 +1135,9 @@ Set environment variables in a step compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"] + with open("script.py", "w") as f: + f.write("print('Hello World!')") + yaml_string = """ kind: pipeline spec: @@ -1517,7 +1147,7 @@ Set environment variables in a step stepDetails: - kind: customScript spec: - description: A step running a notebook + description: A step running a python script infrastructure: kind: infrastructure spec: @@ -1526,82 +1156,28 @@ Set environment variables in a step memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex - name: Notebook_Step + name: Python_Script_Step runtime: kind: runtime spec: conda: - slug: tensorflow26_p37_cpu_v2 + slug: generalml_p37_cpu_v1 type: service - env: - - name: GREETINGS - value: Welcome to OCI Data Science - notebookEncoding: utf-8 - notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb - outputURI: oci://@/ - type: notebook + scriptPathURI: script.py + type: script type: pipeline """.format(compartment_id=compartment_id, project_id=project_id) pipeline = Pipeline.from_yaml(yaml_string) - pipeline.create() - - pipeline_run = pipeline.run() - - - -Watch status update on a pipeline run -===================================== - -.. tabs:: - - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - - pipeline_step = PipelineStep( - name="Python_Script_Step", - description="A step running a python script", - infrastructure=infrastructure, - runtime=runtime - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step], - ) - pipeline.create() pipeline_run = pipeline.run() - + # pipeline_run.show(mode="text") # watch pipeline run status in text - pipeline_run.show(wait=True) # watch pipeline run status in graph - + pipeline_run.show(wait=True) # watch pipeline run status in graph .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os @@ -1646,6 +1222,12 @@ Watch status update on a pipeline run pipeline_run.show(wait=True) # watch pipeline run status in graph + +Monitor logs of a pipeline run +============================== + +.. tabs:: + .. code-tab:: Python3 :caption: YAML @@ -1692,67 +1274,13 @@ Watch status update on a pipeline run pipeline.create() pipeline_run = pipeline.run() - - # pipeline_run.show(mode="text") # watch pipeline run status in text - pipeline_run.show(wait=True) # watch pipeline run status in graph - - - - -Monitor logs of a pipeline run -============================== - -.. tabs:: - - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = ( - CustomScriptStep() - .with_block_storage_size(200) - .with_shape_name("VM.Standard3.Flex") - .with_shape_config_details(ocpus=4, memory_in_gbs=32) - ) - - runtime = ( - ScriptRuntime() - .with_source("script.py") - .with_service_conda("generalml_p37_cpu_v1") - ) - - pipeline_step = PipelineStep( - name="Python_Script_Step", - description="A step running a python script", - infrastructure=infrastructure, - runtime=runtime - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step], - ) - - pipeline.create() - pipeline_run = pipeline.run() # pipeline_run.watch() # stream the consolidated log of the pipeline run pipeline_run.watch(log_type="service_log") # stream service log of the pipeline run pipeline_run.watch("Python_Script_Step", log_type="custom_log") # stream custom log of the step run - .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os @@ -1797,6 +1325,14 @@ Monitor logs of a pipeline run pipeline_run.watch(log_type="service_log") # stream service log of the pipeline run pipeline_run.watch("Python_Script_Step", log_type="custom_log") # stream custom log of the step run + + + +Override configurations when creating a pipeline run +==================================================== + +.. tabs:: + .. code-tab:: Python3 :caption: YAML @@ -1812,6 +1348,9 @@ Monitor logs of a pipeline run yaml_string = """ kind: pipeline spec: + commandLineArguments: argument --key value + environmentVariables: + env: value compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} @@ -1841,58 +1380,6 @@ Monitor logs of a pipeline run pipeline = Pipeline.from_yaml(yaml_string) - pipeline.create() - pipeline_run = pipeline.run() - - # pipeline_run.watch() # stream the consolidated log of the pipeline run - pipeline_run.watch(log_type="service_log") # stream service log of the pipeline run - pipeline_run.watch("Python_Script_Step", log_type="custom_log") # stream custom log of the step run - - -Override configurations when creating a pipeline run -==================================================== - -.. tabs:: - - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - - pipeline_step = PipelineStep( - name="Python_Script_Step", - description="A step running a python script", - infrastructure=infrastructure, - runtime=runtime - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="A single step pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step], - command_line_arguments="argument --key value", - environment_variables={"env": "value"}, - ) - pipeline.create() # Override configurations when creating a pipeline run @@ -1919,12 +1406,10 @@ Override configurations when creating a pipeline run configuration_override_details=configuration_override_details, step_override_details=step_override_details, ) - - - + .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os @@ -1993,80 +1478,7 @@ Override configurations when creating a pipeline run - .. code-tab:: Python3 - :caption: YAML - - from ads.pipeline import Pipeline - import os - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - yaml_string = """ - kind: pipeline - spec: - commandLineArguments: argument --key value - environmentVariables: - env: value - compartmentId: {compartment_id} - displayName: A single step pipeline - projectId: {project_id} - stepDetails: - - kind: customScript - spec: - description: A step running a python script - infrastructure: - kind: infrastructure - spec: - blockStorageSize: 200 - shapeConfigDetails: - memoryInGBs: 32 - ocpus: 4 - shapeName: VM.Standard3.Flex - name: Python_Script_Step - runtime: - kind: runtime - spec: - conda: - slug: generalml_p37_cpu_v1 - type: service - scriptPathURI: script.py - type: script - type: pipeline - """.format(compartment_id=compartment_id, project_id=project_id) - - pipeline = Pipeline.from_yaml(yaml_string) - - pipeline.create() - - # Override configurations when creating a pipeline run - display_override_name = "RunOverrideName" - configuration_override_details = { - "maximum_runtime_in_minutes": 30, - "type": "DEFAULT", - "environment_variables": {"a": "b"}, - "command_line_arguments": "ARGUMENT --KEY VALUE", - } - step_override_details = [ - { - "step_name": "Python_Script_Step", - "step_configuration_details": { - "maximum_runtime_in_minutes": 200, - "environment_variables": {"1": "2"}, - "command_line_arguments": "argument --key value", - }, - } - ] - pipeline_run = pipeline.run( - display_name=display_override_name, - configuration_override_details=configuration_override_details, - step_override_details=step_override_details, - ) - diff --git a/docs/source/user_guide/pipeline/index.rst b/docs/source/user_guide/pipeline/index.rst index 8fc200f8d..43a8ecf72 100644 --- a/docs/source/user_guide/pipeline/index.rst +++ b/docs/source/user_guide/pipeline/index.rst @@ -1,8 +1,8 @@ .. _pipeline: -######### -Pipelines -######### +###################### +Data Science Pipelines +###################### .. versionadded:: 2.8.0 diff --git a/docs/source/user_guide/pipeline/pipeline.rst b/docs/source/user_guide/pipeline/pipeline.rst index 6794d6fe8..6abaf0e78 100644 --- a/docs/source/user_guide/pipeline/pipeline.rst +++ b/docs/source/user_guide/pipeline/pipeline.rst @@ -46,89 +46,6 @@ A ``Pipeline`` instance will be created. .. tabs:: - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import PipelineStep, Pipeline - - pipeline_step_one = PipelineStep( - name="", - description="", - job_id="" - ) - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - runtime = ScriptRuntime( - script_path_uri=oci://@//", - conda={"type": "service", "slug": ""} - ) - - pipeline_step_two = PipelineStep( - name="", - description="", - infrastructure=infrastructure, - runtime=runtime, - ) - - pipeline = Pipeline( - name="", - compartment_id="", - project_id="", - log_group_id="", - log_id="", - enable_service_log=True, # to stream service log in pipeline runs - step_details=[pipeline_step_one, pipeline_step_two], - dag=["pipeline_step_name_1 >> pipeline_step_name_2"], - ) - - .. code-tab:: Python3 - :caption: Python (Alternative) - - from ads.pipeline import PipelineStep, Pipeline - - pipeline_step_one = ( - PipelineStep("") - .with_description("") - .with_job_id("") - ) - - infrastructure = ( - CustomScriptStep() - .with_block_storage_size(200) - .with_shape_name("VM.Standard3.Flex") - .with_shape_config_details(ocpus=4, memory_in_gbs=32) - ) - - runtime = ( - ScriptRuntime() - .with_source("oci://@//") - .with_service_conda("") - ) - - pipeline_step_two = ( - PipelineStep("") - .with_description("") - .with_infrastructure(infrastructure) - .with_runtime(runtime) - ) - - pipeline = ( - Pipeline("") - .with_compartment_id("") - .with_project_id("") - .with_log_group_id("") - .with_log_id("") - .with_enable_service_log(True) # to stream service log in pipeline runs - .with_step_details([pipeline_step_one, pipeline_step_two]) - .with_dag(["pipeline_step_name_1 >> pipeline_step_name_2"]) - ) - - .. code-tab:: Python3 :caption: YAML @@ -177,6 +94,51 @@ A ``Pipeline`` instance will be created. """ pipeline = Pipeline.from_yaml(yaml_string) + + + .. code-tab:: Python3 + :caption: Python + + from ads.pipeline import PipelineStep, Pipeline + + pipeline_step_one = ( + PipelineStep("") + .with_description("") + .with_job_id("") + ) + + infrastructure = ( + CustomScriptStep() + .with_block_storage_size(200) + .with_shape_name("VM.Standard3.Flex") + .with_shape_config_details(ocpus=4, memory_in_gbs=32) + ) + + runtime = ( + ScriptRuntime() + .with_source("oci://@//") + .with_service_conda("") + ) + + pipeline_step_two = ( + PipelineStep("") + .with_description("") + .with_infrastructure(infrastructure) + .with_runtime(runtime) + ) + + pipeline = ( + Pipeline("") + .with_compartment_id("") + .with_project_id("") + .with_log_group_id("") + .with_log_id("") + .with_enable_service_log(True) # to stream service log in pipeline runs + .with_step_details([pipeline_step_one, pipeline_step_two]) + .with_dag(["pipeline_step_name_1 >> pipeline_step_name_2"]) + ) + + Create diff --git a/docs/source/user_guide/pipeline/pipeline_step.rst b/docs/source/user_guide/pipeline/pipeline_step.rst index 472d2332b..b44488b30 100644 --- a/docs/source/user_guide/pipeline/pipeline_step.rst +++ b/docs/source/user_guide/pipeline/pipeline_step.rst @@ -17,28 +17,6 @@ Create a Data Science Job step with the OCID of an existing Job. .. tabs:: - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import PipelineStep - - pipeline_step = PipelineStep( - name="", - description="", - job_id="" - ) - - .. code-tab:: Python3 - :caption: Python (Alternative) - - from ads.pipeline import PipelineStep - - pipeline_step = ( - PipelineStep("") - .with_description("") - .with_job_id("") - ) - .. code-tab:: YAML kind: pipeline @@ -53,7 +31,17 @@ Create a Data Science Job step with the OCID of an existing Job. name: ... type: pipeline + + .. code-tab:: Python3 + :caption: Python + + from ads.pipeline import PipelineStep + pipeline_step = ( + PipelineStep("") + .with_description("") + .with_job_id("") + ) Custom Script Step @@ -67,20 +55,30 @@ When constructing a Custom Scrip step ``infrastructure``, you specify the Comput .. tabs:: - .. code-tab:: Python3 - :caption: Python + .. code-tab:: YAML - from ads.pipeline import CustomScriptStep - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) + kind: pipeline + spec: + ... + stepDetails: + - kind: customScript + spec: + infrastructure: + kind: infrastructure + spec: + blockStorageSize: 200 + shapeConfigDetails: + memoryInGBs: 32 + ocpus: 4 + shapeName: VM.Standard3.Flex + name: Python_Script_Step + ... + type: pipeline .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python from ads.pipeline import CustomScriptStep @@ -104,25 +102,26 @@ To define a Custom Script step with ``GitPythonRuntime`` you can use: .. tabs:: - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import GitPythonRuntime + .. code-tab:: YAML - runtime = GitPythonRuntime( - env={"GREETINGS": "Welcome to OCI Data Science"} - conda={"type": "service", "slug": "pytorch19_p37_gpu_v1"} - url="https://github.com/pytorch/tutorials.git", - entrypoint="beginner_source/examples_nn/polynomial_nn.py", - output_dir="~/Code/tutorials/beginner_source/examples_nn", - outputURI="oci://@/", - ) + kind: runtime + spec: + conda: + slug: pytorch19_p37_gpu_v1 + type: service + entrypoint: beginner_source/examples_nn/polynomial_nn.py + env: + - name: GREETINGS + value: Welcome to OCI Data Science + outputDir: ~/Code/tutorials/beginner_source/examples_nn + outputUri: oci://@/ + url: https://github.com/pytorch/tutorials.git + type: gitPython .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python - from ads.pipeline import GitPythonRuntime runtime = ( @@ -142,22 +141,23 @@ To define a Custom Script step with ``NotebookRuntime`` you can use: .. tabs:: - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import NotebookRuntime - - runtime = NotebookRuntime( - notebook_path_uri="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - notebook_encoding="utf-8", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - env={"GREETINGS": "Welcome to OCI Data Science"} - outputURI="oci://@/", - ) + .. code-tab:: YAML + kind: runtime + spec: + conda: + slug: tensorflow26_p37_cpu_v2 + type: service + env: + - name: GREETINGS + value: Welcome to OCI Data Science + notebookEncoding: utf-8 + notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb + outputURI: oci://bucket_name@namespace/path/to/dir + type: notebook .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python from ads.pipeline import NotebookRuntime @@ -177,23 +177,24 @@ To define a Custom Script step with ``PythonRuntime`` you can use: .. tabs:: - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import PythonRuntime - - runtime = PythonRuntime( - script_path_uri="local/path/to/zip_or_dir", - entrypoint="zip_or_dir/my_package/entry.py", - working_dir="zip_or_dir", - python_path=["my_python_packages"], - output_uri="oci://@/", - conda={"type": "service", "slug": "pytorch19_p37_cpu_v1"} - ) + .. code-tab:: YAML + kind: runtime + spec: + conda: + slug: pytorch19_p37_cpu_v1 + type: service + entrypoint: zip_or_dir/my_package/entry.py + outputDir: output + outputUri: oci://bucket_name@namespace/path/to/dir + pythonPath: + - my_python_packages + scriptPathURI: local/path/to/zip_or_dir + workingDir: zip_or_dir + type: python .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python from ads.pipeline import PythonRuntime @@ -218,19 +219,18 @@ To define a Custom Script step with ``ScriptRuntime`` you can use: .. tabs:: - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import ScriptRuntime - - runtime = ScriptRuntime( - script_path_uri="oci://@//", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - + .. code-tab:: YAML + + kind: runtime + spec: + conda: + slug: tensorflow26_p37_cpu_v2 + type: service + scriptPathURI: oci://@// + type: script .. code-tab:: Python3 - :caption: Python (Alternative) + :caption: Python from ads.pipeline import ScriptRuntime @@ -244,32 +244,6 @@ With ``Infrastructure`` and ``runtime`` provided, create a pipeline step of the .. tabs:: - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import PipelineStep - - pipeline_step = PipelineStep( - name="", - description="", - infrastructure=infrastructure, - runtime=runtime, - ) - - - .. code-tab:: Python3 - :caption: Python (Alternative) - - from ads.pipeline import PipelineStep - - pipeline_step = ( - PipelineStep("") - .with_description("") - .with_infrastructure(infrastructure) - .with_runtime(runtime) - ) - - .. code-tab:: YAML kind: pipeline @@ -299,6 +273,17 @@ With ``Infrastructure`` and ``runtime`` provided, create a pipeline step of the type: script ... type: pipeline + + .. code-tab:: Python3 + :caption: Python + + from ads.pipeline import PipelineStep + pipeline_step = ( + PipelineStep("") + .with_description("") + .with_infrastructure(infrastructure) + .with_runtime(runtime) + ) diff --git a/docs/source/user_guide/pipeline/quick_start.rst b/docs/source/user_guide/pipeline/quick_start.rst index 05ce53de2..b335ff894 100644 --- a/docs/source/user_guide/pipeline/quick_start.rst +++ b/docs/source/user_guide/pipeline/quick_start.rst @@ -15,143 +15,6 @@ The following example shows creating and runnning a pipeline with multiple steps .. tabs:: - .. code-tab:: Python3 - :caption: Python - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = CustomScriptStep( - block_storage_size=200, - shape_name="VM.Standard3.Flex", - shape_config_details={"ocpus": 4, "memory_in_gbs": 32}, - ) - - script_runtime = ScriptRuntime( - script_path_uri="script.py", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - - notebook_runtime = NotebookRuntime( - notebook_path_uri="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - conda={"type": "service", "slug": "tensorflow26_p37_cpu_v2"} - ) - - pipeline_step_1 = PipelineStep( - name="step_1", - description="A step running a python script", - infrastructure=infrastructure, - runtime=script_runtime - ) - - pipeline_step_2 = PipelineStep( - name="step_2", - description="A step running a notebook", - infrastructure=infrastructure, - runtime=notebook_runtime - ) - - pipeline_step_3 = PipelineStep( - name="step_3", - description="A step running a python script", - infrastructure=infrastructure, - runtime=script_runtime - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = Pipeline( - name="An example pipeline", - compartment_id=compartment_id, - project_id=project_id, - step_details=[pipeline_step_1, pipeline_step_2, pipeline_step_3], - dag=["(step_1, step_2) >> step_3"], - ) - - pipeline.create() # create the pipeline - pipeline.show() # visualize the pipeline - - pipeline_run = pipeline.run() # run the pipeline - - pipeline_run.show() # watch the pipeline run status - - - - .. code-tab:: Python3 - :caption: Python (Alternative) - - from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime - import os - - with open("script.py", "w") as f: - f.write("print('Hello World!')") - - infrastructure = ( - CustomScriptStep() - .with_block_storage_size(200) - .with_shape_name("VM.Standard3.Flex") - .with_shape_config_details(ocpus=4, memory_in_gbs=32) - ) - - script_runtime = ( - ScriptRuntime() - .with_source("script.py") - .with_service_conda("generalml_p37_cpu_v1") - ) - - notebook_runtime = ( - NotebookRuntime() - .with_notebook( - path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", - encoding='utf-8' - ) - .with_service_conda("tensorflow26_p37_cpu_v2") - ) - - pipeline_step_1 = ( - PipelineStep("step_1") - .with_description("A step running a python script") - .with_infrastructure(infrastructure) - .with_runtime(script_runtime) - ) - - pipeline_step_2 = ( - PipelineStep("step_2") - .with_description("A step running a notebook") - .with_infrastructure(infrastructure) - .with_runtime(notebook_runtime) - ) - - pipeline_step_3 = ( - PipelineStep("step_3") - .with_description("A step running a python script") - .with_infrastructure(infrastructure) - .with_runtime(script_runtime) - ) - - compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] - project_id = os.environ["PROJECT_OCID"] - - pipeline = ( - Pipeline("An example pipeline") - .with_compartment_id(compartment_id) - .with_project_id(project_id) - .with_step_details([pipeline_step_1, pipeline_step_2, pipeline_step_3]) - .with_dag(["(step_1, step_2) >> step_3"]) - ) - - pipeline.create() # create the pipeline - pipeline.show() # visualize the pipeline - - pipeline_run = pipeline.run() # run the pipeline - - pipeline_run.show() # watch the pipeline run status - - .. code-tab:: Python3 :caption: YAML @@ -247,6 +110,77 @@ The following example shows creating and runnning a pipeline with multiple steps pipeline_run.show() # watch the pipeline run status + .. code-tab:: Python3 + :caption: Python + + from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime + import os + + with open("script.py", "w") as f: + f.write("print('Hello World!')") + + infrastructure = ( + CustomScriptStep() + .with_block_storage_size(200) + .with_shape_name("VM.Standard3.Flex") + .with_shape_config_details(ocpus=4, memory_in_gbs=32) + ) + + script_runtime = ( + ScriptRuntime() + .with_source("script.py") + .with_service_conda("generalml_p37_cpu_v1") + ) + + notebook_runtime = ( + NotebookRuntime() + .with_notebook( + path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", + encoding='utf-8' + ) + .with_service_conda("tensorflow26_p37_cpu_v2") + ) + + pipeline_step_1 = ( + PipelineStep("step_1") + .with_description("A step running a python script") + .with_infrastructure(infrastructure) + .with_runtime(script_runtime) + ) + + pipeline_step_2 = ( + PipelineStep("step_2") + .with_description("A step running a notebook") + .with_infrastructure(infrastructure) + .with_runtime(notebook_runtime) + ) + + pipeline_step_3 = ( + PipelineStep("step_3") + .with_description("A step running a python script") + .with_infrastructure(infrastructure) + .with_runtime(script_runtime) + ) + + compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] + project_id = os.environ["PROJECT_OCID"] + + pipeline = ( + Pipeline("An example pipeline") + .with_compartment_id(compartment_id) + .with_project_id(project_id) + .with_step_details([pipeline_step_1, pipeline_step_2, pipeline_step_3]) + .with_dag(["(step_1, step_2) >> step_3"]) + ) + + pipeline.create() # create the pipeline + pipeline.show() # visualize the pipeline + + pipeline_run = pipeline.run() # run the pipeline + + pipeline_run.show() # watch the pipeline run status + + ADS CLI =======