Migrate Dataproc Metastore system tests according to AIP-47 (#26858)

airflow/providers/google/cloud/hooks/dataproc_metastore.py

@@ -116,7 +116,7 @@ def create_metadata_import(
         project_id: str,
         region: str,
         service_id: str,
-        metadata_import: MetadataImport,
+        metadata_import: dict | MetadataImport,
         metadata_import_id: str,
         request_id: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,

airflow/providers/google/cloud/operators/dataproc.py

@@ -2128,7 +2128,7 @@ def execute(self, context: Context):
                 metadata=self.metadata,
             )
 
-            # The existing batch may be a in a number of states other than 'SUCCEEDED'
+            # The existing batch may be a number of states other than 'SUCCEEDED'
             if result.state != Batch.State.SUCCEEDED:
                 if result.state == Batch.State.FAILED or result.state == Batch.State.CANCELLED:
                     raise AirflowException(

airflow/providers/google/cloud/operators/dataproc_metastore.py

@@ -311,7 +311,7 @@ def __init__(
         project_id: str,
         region: str,
         service_id: str,
-        metadata_import: MetadataImport,
+        metadata_import: dict | MetadataImport,
         metadata_import_id: str,
         request_id: str | None = None,
         retry: Retry | _MethodDefault = DEFAULT,

docs/apache-airflow-providers-google/operators/cloud/dataproc_metastore.rst

@@ -33,7 +33,7 @@ For more information about the available fields to pass when creating a service,
 
 A simple service configuration can look as followed:
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py
     :language: python
     :dedent: 0
     :start-after: [START how_to_cloud_dataproc_metastore_create_service]
@@ -42,7 +42,7 @@ A simple service configuration can look as followed:
 With this configuration we can create the service:
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreCreateServiceOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_create_service_operator]
@@ -55,7 +55,7 @@ To get a service you can use:
 
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreGetServiceOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_get_service_operator]
@@ -69,7 +69,7 @@ For more information on updateMask and other parameters take a look at `Dataproc
 
 An example of a new service config and the updateMask:
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py
     :language: python
     :dedent: 0
     :start-after: [START how_to_cloud_dataproc_metastore_update_service]
@@ -78,7 +78,7 @@ An example of a new service config and the updateMask:
 To update a service you can use:
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreUpdateServiceOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_update_service_operator]
@@ -91,7 +91,7 @@ To delete a service you can use:
 
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreDeleteServiceOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_delete_service_operator]
@@ -104,7 +104,7 @@ To export metadata you can use:
 
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreExportMetadataOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_export_metadata_operator]
@@ -117,7 +117,7 @@ To restore a service you can use:
 
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreRestoreServiceOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_restore_service_operator]
@@ -131,7 +131,7 @@ For more information about the available fields to pass when creating a metadata
 
 A simple metadata import configuration can look as followed:
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py
     :language: python
     :dedent: 0
     :start-after: [START how_to_cloud_dataproc_metastore_create_metadata_import]
@@ -140,7 +140,7 @@ A simple metadata import configuration can look as followed:
 To create a metadata import you can use:
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreCreateMetadataImportOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_create_metadata_import_operator]
@@ -154,7 +154,7 @@ For more information about the available fields to pass when creating a backup,
 
 A simple backup configuration can look as followed:
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py
     :language: python
     :dedent: 0
     :start-after: [START how_to_cloud_dataproc_metastore_create_backup]
@@ -163,7 +163,7 @@ A simple backup configuration can look as followed:
 With this configuration we can create the backup:
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreCreateBackupOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_create_backup_operator]
@@ -176,7 +176,7 @@ To delete a backup you can use:
 
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreDeleteBackupOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_delete_backup_operator]
@@ -189,7 +189,7 @@ To list backups you can use:
 
 :class:`~airflow.providers.google.cloud.operators.dataproc_metastore.DataprocMetastoreListBackupsOperator`
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc_metastore.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/dataproc_metastore/example_dataproc_metastore_backup.py
     :language: python
     :dedent: 4
     :start-after: [START how_to_cloud_dataproc_metastore_list_backups_operator]

tests/system/providers/google/cloud/dataproc_metastore/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.