examples.rst

Examples

Create a pipeline

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Python_Script_Step runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

pipeline_step = (

PipelineStep("Python_Script_Step") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step])

)

pipeline.create()

Run a job as a step

Python3

rom ads.jobs import Job, DataScienceJob, ScriptRuntime from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

DataScienceJob() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

job = (

Job() .with_infrastructure(infrastructure) .with_runtime(runtime)

) job.create() # create a job

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: dataScienceJob spec: description: A step running a job jobId: {job_id} name: Job_Step type: pipeline """.format(compartment_id=compartment_id, project_id=project_id, job_id=job.id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()

Python3

from ads.jobs import Job, DataScienceJob, ScriptRuntime from ads.pipeline import Pipeline, PipelineStep import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

DataScienceJob() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

job = (

Job() .with_infrastructure(infrastructure) .with_runtime(runtime)

) job.create() # create a job

pipeline_step = (

PipelineStep("Job_Step") .with_description("A step running a job") .with_job_id(job.id)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step])

)

pipeline.create()

pipeline_run = pipeline.run()

Run a python script as a step

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Python_Script_Step runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

pipeline_step = (

PipelineStep("Python_Script_Step") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step])

)

pipeline.create()

pipeline_run = pipeline.run()

Run a notebook as a step

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: customScript spec: description: A step running a notebook infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Notebook_Step runtime: kind: runtime spec: conda: slug: tensorflow26_p37_cpu_v2 type: service env: - name: GREETINGS value: Welcome to OCI Data Science notebookEncoding: utf-8 notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb outputURI: oci://<bucket_name>@<namespace>/<prefix> type: notebook type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, NotebookRuntime import os

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

runtime = (

NotebookRuntime() .with_notebook( path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", encoding='utf-8' ) .with_service_conda("tensorflow26_p37_cpu_v2") .with_environment_variable(GREETINGS="Welcome to OCI Data Science") .with_output("oci://<bucket_name>@<namespace>/<prefix>")

)

pipeline_step = (

PipelineStep("Notebook_Step") .with_description("A step running a notebook") .with_infrastructure(infrastructure) .with_runtime(runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step])

)

pipeline.create()

pipeline_run = pipeline.run()

Run two steps with the same infrastructure

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Python_Script_Step runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script - kind: customScript spec: description: A step running a notebook infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Notebook_Step runtime: kind: runtime spec: conda: slug: tensorflow26_p37_cpu_v2 type: service env: - name: GREETINGS value: Welcome to OCI Data Science notebookEncoding: utf-8 notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb outputURI: oci://<bucket_name>@<namespace>/<prefix> type: notebook type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

step_one_runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

pipeline_step_one = (

PipelineStep("Python_Script_Step") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(step_one_runtime)

)

step_two_runtime = (

NotebookRuntime() .with_notebook( path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", encoding='utf-8' ) .with_service_conda("tensorflow26_p37_cpu_v2") .with_environment_variable(GREETINGS="Welcome to OCI Data Science") .with_output("oci://<bucket_name>@<namespace>/<prefix>")

)

pipeline_step_two = (

PipelineStep("Notebook_Step") .with_description("A step running a notebook") .with_infrastructure(infrastructure) .with_runtime(step_two_runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step_one, pipeline_step_two])

)

pipeline.create()

pipeline_run = pipeline.run()

Run two steps in parallel

In the example below, when DAG is not specified, the steps in the pipeline run in parallel.

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Python_Script_Step runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script - kind: customScript spec: description: A step running a notebook infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Notebook_Step runtime: kind: runtime spec: conda: slug: tensorflow26_p37_cpu_v2 type: service env: - name: GREETINGS value: Welcome to OCI Data Science notebookEncoding: utf-8 notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb outputURI: oci://<bucket_name>@<namespace>/<prefix> type: notebook type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

step_one_runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

pipeline_step_one = (

PipelineStep("Python_Script_Step") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(step_one_runtime)

)

step_two_runtime = (

NotebookRuntime() .with_notebook( path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", encoding='utf-8' ) .with_service_conda("tensorflow26_p37_cpu_v2") .with_environment_variable(GREETINGS="Welcome to OCI Data Science") .with_output("oci://<bucket_name>@<namespace>/<prefix>")

)

pipeline_step_two = (

PipelineStep("Notebook_Step") .with_description("A step running a notebook") .with_infrastructure(infrastructure) .with_runtime(step_two_runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step_one, pipeline_step_two])

)

pipeline.create()

pipeline_run = pipeline.run()

Run two steps sequentially

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} dag: - Python_Script_Step >> Notebook_Step stepDetails: - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Python_Script_Step runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script - kind: customScript spec: description: A step running a notebook infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Notebook_Step runtime: kind: runtime spec: conda: slug: tensorflow26_p37_cpu_v2 type: service env: - name: GREETINGS value: Welcome to OCI Data Science notebookEncoding: utf-8 notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb outputURI: oci://<bucket_name>@<namespace>/<prefix> type: notebook type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

step_one_runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

pipeline_step_one = (

PipelineStep("Python_Script_Step") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(step_one_runtime)

)

step_two_runtime = (

NotebookRuntime() .with_notebook( path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", encoding='utf-8' ) .with_service_conda("tensorflow26_p37_cpu_v2") .with_environment_variable(GREETINGS="Welcome to OCI Data Science") .with_output("oci://<bucket_name>@<namespace>/<prefix>")

)

pipeline_step_two = (

PipelineStep("Notebook_Step") .with_description("A step running a notebook") .with_infrastructure(infrastructure) .with_runtime(step_two_runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step_one, pipeline_step_two]) .with_dag(["Python_Script_Step >> Notebook_Step"])

)

pipeline.create()

pipeline_run = pipeline.run()

Run multiple steps with dependencies specified in DAG

In this example, step_1 and step_2 run in parallel and step_3 runs after step_1 and step_2 are complete.

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: An example pipeline projectId: {project_id} dag: - (step_1, step_2) >> step_3 stepDetails: - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: step_1 runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script - kind: customScript spec: description: A step running a notebook infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: step_2 runtime: kind: runtime spec: conda: slug: tensorflow26_p37_cpu_v2 type: service notebookEncoding: utf-8 notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb type: notebook - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: step_3 runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create() # create the pipeline pipeline.show() # visualize the pipeline

pipeline_run = pipeline.run() # run the pipeline

pipeline_run.show(wait=True) # watch the pipeline run status

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

script_runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

notebook_runtime = (

NotebookRuntime() .with_notebook( path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", encoding='utf-8' ) .with_service_conda("tensorflow26_p37_cpu_v2")

)

pipeline_step_1 = (

PipelineStep("step_1") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(script_runtime)

)

pipeline_step_2 = (

PipelineStep("step_2") .with_description("A step running a notebook") .with_infrastructure(infrastructure) .with_runtime(notebook_runtime)

)

pipeline_step_3 = (

PipelineStep("step_3") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(script_runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("An example pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step_1, pipeline_step_2, pipeline_step_3]) .with_dag(["(step_1, step_2) >> step_3"])

)

pipeline.create() # create the pipeline pipeline.show() # visualize the pipeline

pipeline_run = pipeline.run() # run the pipeline

pipeline_run.show(wait=True) # watch the pipeline run status

Set environment variables in a step

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: customScript spec: description: A step running a notebook infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Notebook_Step runtime: kind: runtime spec: conda: slug: tensorflow26_p37_cpu_v2 type: service env: - name: GREETINGS value: Welcome to OCI Data Science notebookEncoding: utf-8 notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb outputURI: oci://<bucket_name>@<namespace>/<prefix> type: notebook type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, NotebookRuntime import os

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

runtime = (

NotebookRuntime() .with_notebook( path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb", encoding='utf-8' ) .with_service_conda("tensorflow26_p37_cpu_v2") .with_environment_variable(GREETINGS="Welcome to OCI Data Science") .with_output("oci://<bucket_name>@<namespace>/<prefix>")

)

pipeline_step = (

PipelineStep("Notebook_Step") .with_description("A step running a notebook") .with_infrastructure(infrastructure) .with_runtime(runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step])

)

pipeline.create()

pipeline_run = pipeline.run()

Watch status update on a pipeline run

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Python_Script_Step runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create() pipeline_run = pipeline.run()

# pipeline_run.show(mode="text") # watch pipeline run status in text pipeline_run.show(wait=True) # watch pipeline run status in graph

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

pipeline_step = (

PipelineStep("Python_Script_Step") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step])

)

pipeline.create() pipeline_run = pipeline.run()

# pipeline_run.show(mode="text") # watch pipeline run status in text pipeline_run.show(wait=True) # watch pipeline run status in graph

Monitor logs of a pipeline run

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

yaml_string = """ kind: pipeline spec: compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Python_Script_Step runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create() pipeline_run = pipeline.run()

# pipeline_run.watch() # stream the consolidated log of the pipeline run pipeline_run.watch(log_type="service_log") # stream service log of the pipeline run pipeline_run.watch("Python_Script_Step", log_type="custom_log") # stream custom log of the step run

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

pipeline_step = (

PipelineStep("Python_Script_Step") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step])

)

pipeline.create() pipeline_run = pipeline.run()

# pipeline_run.watch() # stream the consolidated log of the pipeline run pipeline_run.watch(log_type="service_log") # stream service log of the pipeline run pipeline_run.watch("Python_Script_Step", log_type="custom_log") # stream custom log of the step run

Override configurations when creating a pipeline run

Python3

from ads.pipeline import Pipeline import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:

f.write("print('Hello World!')")

yaml_string = """ kind: pipeline spec: commandLineArguments: argument --key value environmentVariables: env: value compartmentId: {compartment_id} displayName: A single step pipeline projectId: {project_id} stepDetails: - kind: customScript spec: description: A step running a python script infrastructure: kind: infrastructure spec: blockStorageSize: 200 shapeConfigDetails: memoryInGBs: 32 ocpus: 4 shapeName: VM.Standard3.Flex name: Python_Script_Step runtime: kind: runtime spec: conda: slug: generalml_p37_cpu_v1 type: service scriptPathURI: script.py type: script type: pipeline """.format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

# Override configurations when creating a pipeline run display_override_name = "RunOverrideName" configuration_override_details = { "maximum_runtime_in_minutes": 30, "type": "DEFAULT", "environment_variables": {"a": "b"}, "command_line_arguments": "ARGUMENT --KEY VALUE", }

step_override_details = [ { "step_name": "Python_Script_Step", "step_configuration_details": { "maximum_runtime_in_minutes": 200, "environment_variables": {"1": "2"}, "command_line_arguments": "argument --key value", }, } ] pipeline_run = pipeline.run( display_name=display_override_name, configuration_override_details=configuration_override_details, step_override_details=step_override_details, )

Python3

from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime import os

with open("script.py", "w") as f:

f.write("print('Hello World!')")

infrastructure = (

CustomScriptStep() .with_block_storage_size(200) .with_shape_name("VM.Standard3.Flex") .with_shape_config_details(ocpus=4, memory_in_gbs=32)

)

runtime = (

ScriptRuntime() .with_source("script.py") .with_service_conda("generalml_p37_cpu_v1")

)

pipeline_step = (

PipelineStep("Python_Script_Step") .with_description("A step running a python script") .with_infrastructure(infrastructure) .with_runtime(runtime)

)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID'] project_id = os.environ["PROJECT_OCID"]

pipeline = (

Pipeline("A single step pipeline") .with_compartment_id(compartment_id) .with_project_id(project_id) .with_step_details([pipeline_step]) .with_argument("argument", key="value") .with_environment_variable(env="value")

)

pipeline.create()

# Override configurations when creating a pipeline run display_override_name = "RunOverrideName" configuration_override_details = { "maximum_runtime_in_minutes": 30, "type": "DEFAULT", "environment_variables": {"a": "b"}, "command_line_arguments": "ARGUMENT --KEY VALUE", }

step_override_details = [ { "step_name": "Python_Script_Step", "step_configuration_details": { "maximum_runtime_in_minutes": 200, "environment_variables": {"1": "2"}, "command_line_arguments": "argument --key value", }, } ] pipeline_run = pipeline.run( display_name=display_override_name, configuration_override_details=configuration_override_details, step_override_details=step_override_details, )