diff --git a/airflow/providers/google/cloud/example_dags/example_dataproc.py b/airflow/providers/google/cloud/example_dags/example_dataproc.py index 87acf3f66b6f..debb95cffb2c 100644 --- a/airflow/providers/google/cloud/example_dags/example_dataproc.py +++ b/airflow/providers/google/cloud/example_dags/example_dataproc.py @@ -33,6 +33,7 @@ DataprocDeleteBatchOperator, DataprocDeleteClusterOperator, DataprocGetBatchOperator, + DataprocInstantiateInlineWorkflowTemplateOperator, DataprocInstantiateWorkflowTemplateOperator, DataprocListBatchesOperator, DataprocSubmitJobOperator, @@ -256,6 +257,12 @@ ) # [END how_to_cloud_dataproc_trigger_workflow_template] + # [START how_to_cloud_dataproc_instantiate_inline_workflow_template] + instantiate_inline_workflow_template = DataprocInstantiateInlineWorkflowTemplateOperator( + task_id='instantiate_inline_workflow_template', template=WORKFLOW_TEMPLATE, region=REGION + ) + # [END how_to_cloud_dataproc_instantiate_inline_workflow_template] + pig_task = DataprocSubmitJobOperator( task_id="pig_task", job=PIG_JOB, region=REGION, project_id=PROJECT_ID ) diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index 753ca287132d..7cf833357838 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -1673,8 +1673,11 @@ class DataprocInstantiateInlineWorkflowTemplateOperator(BaseOperator): wait until the WorkflowTemplate is finished executing. .. seealso:: - Please refer to: - https://cloud.google.com/dataproc/docs/reference/rest/v1beta2/projects.regions.workflowTemplates/instantiateInline + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:DataprocInstantiateInlineWorkflowTemplateOperator` + + For more detail on about instantiate inline have a look at the reference: + https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.workflowTemplates/instantiateInline :param template: The template contents. (templated) :param project_id: The ID of the google cloud project in which diff --git a/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst b/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst index c4339fd8ff25..1909249258dc 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst @@ -32,6 +32,7 @@ Prerequisite Tasks .. _howto/operator:DataprocCreateClusterOperator: +.. _howto/operator:DataprocInstantiateInlineWorkflowTemplateOperator: Create a Cluster ---------------- @@ -212,6 +213,15 @@ Once a workflow is created users can trigger it using :start-after: [START how_to_cloud_dataproc_trigger_workflow_template] :end-before: [END how_to_cloud_dataproc_trigger_workflow_template] +The inline operator is an alternative. It creates a workflow, run it, and delete it afterwards: +:class:`~airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateInlineWorkflowTemplateOperator`: + +.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc.py + :language: python + :dedent: 4 + :start-after: [START how_to_cloud_dataproc_instantiate_inline_workflow_template] + :end-before: [END how_to_cloud_dataproc_instantiate_inline_workflow_template] + Create a Batch -------------- diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py index 3200f39ffe94..9c019f418744 100644 --- a/tests/always/test_project_structure.py +++ b/tests/always/test_project_structure.py @@ -192,7 +192,6 @@ class TestGoogleProviderProjectStructure(unittest.TestCase): # Please at the examples to those operators at the earliest convenience :) MISSING_EXAMPLES_FOR_OPERATORS = { - 'airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateInlineWorkflowTemplateOperator', 'airflow.providers.google.cloud.operators.mlengine.MLEngineTrainingCancelJobOperator', 'airflow.providers.google.cloud.operators.dlp.CloudDLPGetStoredInfoTypeOperator', 'airflow.providers.google.cloud.operators.dlp.CloudDLPReidentifyContentOperator',