feat!: add support for Virtual Dataproc cluster running on GKE cluster (

#344) - [ ] Regenerate this pull request now. Committer: @Padmaar PiperOrigin-RevId: 429111624 Source-Link: googleapis/googleapis@da999a2 Source-Link: googleapis/googleapis-gen@99c5b3e Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiOTljNWIzZTk4YmFhMWRlOTM3NzZhYTRiNWNkNGM3MzYxMzUzZTRmNiJ9 fix: remove namespaced_gke_deployment_target fix: move GkeClusterConfig to google.cloud.dataproc_v1.types.shared
googleapis · Feb 25, 2022 · 116077b · 116077b
1 parent 4344dfa
commit 116077b
Show file tree

Hide file tree

Showing 11 changed files with 402 additions and 162 deletions.
diff --git a/google/cloud/dataproc/__init__.py b/google/cloud/dataproc/__init__.py
@@ -83,6 +83,7 @@
 from google.cloud.dataproc_v1.types.batches import SparkSqlBatch
 from google.cloud.dataproc_v1.types.clusters import AcceleratorConfig
 from google.cloud.dataproc_v1.types.clusters import AutoscalingConfig
+from google.cloud.dataproc_v1.types.clusters import AuxiliaryServicesConfig
 from google.cloud.dataproc_v1.types.clusters import Cluster
 from google.cloud.dataproc_v1.types.clusters import ClusterConfig
 from google.cloud.dataproc_v1.types.clusters import ClusterMetrics
@@ -97,7 +98,6 @@
 from google.cloud.dataproc_v1.types.clusters import EndpointConfig
 from google.cloud.dataproc_v1.types.clusters import GceClusterConfig
 from google.cloud.dataproc_v1.types.clusters import GetClusterRequest
-from google.cloud.dataproc_v1.types.clusters import GkeClusterConfig
 from google.cloud.dataproc_v1.types.clusters import IdentityConfig
 from google.cloud.dataproc_v1.types.clusters import InstanceGroupConfig
 from google.cloud.dataproc_v1.types.clusters import KerberosConfig
@@ -115,6 +115,7 @@
 from google.cloud.dataproc_v1.types.clusters import StartClusterRequest
 from google.cloud.dataproc_v1.types.clusters import StopClusterRequest
 from google.cloud.dataproc_v1.types.clusters import UpdateClusterRequest
+from google.cloud.dataproc_v1.types.clusters import VirtualClusterConfig
 from google.cloud.dataproc_v1.types.jobs import CancelJobRequest
 from google.cloud.dataproc_v1.types.jobs import DeleteJobRequest
 from google.cloud.dataproc_v1.types.jobs import GetJobRequest
@@ -144,6 +145,11 @@
 from google.cloud.dataproc_v1.types.operations import ClusterOperationStatus
 from google.cloud.dataproc_v1.types.shared import EnvironmentConfig
 from google.cloud.dataproc_v1.types.shared import ExecutionConfig
+from google.cloud.dataproc_v1.types.shared import GkeClusterConfig
+from google.cloud.dataproc_v1.types.shared import GkeNodePoolConfig
+from google.cloud.dataproc_v1.types.shared import GkeNodePoolTarget
+from google.cloud.dataproc_v1.types.shared import KubernetesClusterConfig
+from google.cloud.dataproc_v1.types.shared import KubernetesSoftwareConfig
 from google.cloud.dataproc_v1.types.shared import PeripheralsConfig
 from google.cloud.dataproc_v1.types.shared import RuntimeConfig
 from google.cloud.dataproc_v1.types.shared import RuntimeInfo
@@ -219,6 +225,7 @@
     "SparkSqlBatch",
     "AcceleratorConfig",
     "AutoscalingConfig",
+    "AuxiliaryServicesConfig",
     "Cluster",
     "ClusterConfig",
     "ClusterMetrics",
@@ -233,7 +240,6 @@
     "EndpointConfig",
     "GceClusterConfig",
     "GetClusterRequest",
-    "GkeClusterConfig",
     "IdentityConfig",
     "InstanceGroupConfig",
     "KerberosConfig",
@@ -251,6 +257,7 @@
     "StartClusterRequest",
     "StopClusterRequest",
     "UpdateClusterRequest",
+    "VirtualClusterConfig",
     "CancelJobRequest",
     "DeleteJobRequest",
     "GetJobRequest",
@@ -280,6 +287,11 @@
     "ClusterOperationStatus",
     "EnvironmentConfig",
     "ExecutionConfig",
+    "GkeClusterConfig",
+    "GkeNodePoolConfig",
+    "GkeNodePoolTarget",
+    "KubernetesClusterConfig",
+    "KubernetesSoftwareConfig",
     "PeripheralsConfig",
     "RuntimeConfig",
     "RuntimeInfo",

diff --git a/google/cloud/dataproc_v1/__init__.py b/google/cloud/dataproc_v1/__init__.py
@@ -47,6 +47,7 @@
 from .types.batches import SparkSqlBatch
 from .types.clusters import AcceleratorConfig
 from .types.clusters import AutoscalingConfig
+from .types.clusters import AuxiliaryServicesConfig
 from .types.clusters import Cluster
 from .types.clusters import ClusterConfig
 from .types.clusters import ClusterMetrics
@@ -61,7 +62,6 @@
 from .types.clusters import EndpointConfig
 from .types.clusters import GceClusterConfig
 from .types.clusters import GetClusterRequest
-from .types.clusters import GkeClusterConfig
 from .types.clusters import IdentityConfig
 from .types.clusters import InstanceGroupConfig
 from .types.clusters import KerberosConfig
@@ -79,6 +79,7 @@
 from .types.clusters import StartClusterRequest
 from .types.clusters import StopClusterRequest
 from .types.clusters import UpdateClusterRequest
+from .types.clusters import VirtualClusterConfig
 from .types.jobs import CancelJobRequest
 from .types.jobs import DeleteJobRequest
 from .types.jobs import GetJobRequest
@@ -108,6 +109,11 @@
 from .types.operations import ClusterOperationStatus
 from .types.shared import EnvironmentConfig
 from .types.shared import ExecutionConfig
+from .types.shared import GkeClusterConfig
+from .types.shared import GkeNodePoolConfig
+from .types.shared import GkeNodePoolTarget
+from .types.shared import KubernetesClusterConfig
+from .types.shared import KubernetesSoftwareConfig
 from .types.shared import PeripheralsConfig
 from .types.shared import RuntimeConfig
 from .types.shared import RuntimeInfo
@@ -146,6 +152,7 @@
     "AutoscalingConfig",
     "AutoscalingPolicy",
     "AutoscalingPolicyServiceClient",
+    "AuxiliaryServicesConfig",
     "BasicAutoscalingAlgorithm",
     "BasicYarnAutoscalingConfig",
     "Batch",
@@ -187,6 +194,8 @@
     "GetJobRequest",
     "GetWorkflowTemplateRequest",
     "GkeClusterConfig",
+    "GkeNodePoolConfig",
+    "GkeNodePoolTarget",
     "HadoopJob",
     "HiveJob",
     "IdentityConfig",
@@ -202,6 +211,8 @@
     "JobScheduling",
     "JobStatus",
     "KerberosConfig",
+    "KubernetesClusterConfig",
+    "KubernetesSoftwareConfig",
     "LifecycleConfig",
     "ListAutoscalingPoliciesRequest",
     "ListAutoscalingPoliciesResponse",
@@ -250,6 +261,7 @@
     "UpdateJobRequest",
     "UpdateWorkflowTemplateRequest",
     "ValueValidation",
+    "VirtualClusterConfig",
     "WorkflowGraph",
     "WorkflowMetadata",
     "WorkflowNode",

diff --git a/google/cloud/dataproc_v1/services/cluster_controller/async_client.py b/google/cloud/dataproc_v1/services/cluster_controller/async_client.py
@@ -53,8 +53,6 @@ class ClusterControllerAsyncClient:
     DEFAULT_ENDPOINT = ClusterControllerClient.DEFAULT_ENDPOINT
     DEFAULT_MTLS_ENDPOINT = ClusterControllerClient.DEFAULT_MTLS_ENDPOINT
 
-    cluster_path = staticmethod(ClusterControllerClient.cluster_path)
-    parse_cluster_path = staticmethod(ClusterControllerClient.parse_cluster_path)
     service_path = staticmethod(ClusterControllerClient.service_path)
     parse_service_path = staticmethod(ClusterControllerClient.parse_service_path)
     common_billing_account_path = staticmethod(

diff --git a/google/cloud/dataproc_v1/services/cluster_controller/client.py b/google/cloud/dataproc_v1/services/cluster_controller/client.py
@@ -168,22 +168,6 @@ def transport(self) -> ClusterControllerTransport:
         """
         return self._transport
 
-    @staticmethod
-    def cluster_path(project: str, location: str, cluster: str,) -> str:
-        """Returns a fully-qualified cluster string."""
-        return "projects/{project}/locations/{location}/clusters/{cluster}".format(
-            project=project, location=location, cluster=cluster,
-        )
-
-    @staticmethod
-    def parse_cluster_path(path: str) -> Dict[str, str]:
-        """Parses a cluster path into its component segments."""
-        m = re.match(
-            r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/clusters/(?P<cluster>.+?)$",
-            path,
-        )
-        return m.groupdict() if m else {}
-
     @staticmethod
     def service_path(project: str, location: str, service: str,) -> str:
         """Returns a fully-qualified service string."""

diff --git a/google/cloud/dataproc_v1/services/workflow_template_service/async_client.py b/google/cloud/dataproc_v1/services/workflow_template_service/async_client.py
@@ -53,8 +53,6 @@ class WorkflowTemplateServiceAsyncClient:
     DEFAULT_ENDPOINT = WorkflowTemplateServiceClient.DEFAULT_ENDPOINT
     DEFAULT_MTLS_ENDPOINT = WorkflowTemplateServiceClient.DEFAULT_MTLS_ENDPOINT
 
-    cluster_path = staticmethod(WorkflowTemplateServiceClient.cluster_path)
-    parse_cluster_path = staticmethod(WorkflowTemplateServiceClient.parse_cluster_path)
     service_path = staticmethod(WorkflowTemplateServiceClient.service_path)
     parse_service_path = staticmethod(WorkflowTemplateServiceClient.parse_service_path)
     workflow_template_path = staticmethod(

diff --git a/google/cloud/dataproc_v1/services/workflow_template_service/client.py b/google/cloud/dataproc_v1/services/workflow_template_service/client.py
@@ -168,22 +168,6 @@ def transport(self) -> WorkflowTemplateServiceTransport:
         """
         return self._transport
 
-    @staticmethod
-    def cluster_path(project: str, location: str, cluster: str,) -> str:
-        """Returns a fully-qualified cluster string."""
-        return "projects/{project}/locations/{location}/clusters/{cluster}".format(
-            project=project, location=location, cluster=cluster,
-        )
-
-    @staticmethod
-    def parse_cluster_path(path: str) -> Dict[str, str]:
-        """Parses a cluster path into its component segments."""
-        m = re.match(
-            r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/clusters/(?P<cluster>.+?)$",
-            path,
-        )
-        return m.groupdict() if m else {}
-
     @staticmethod
     def service_path(project: str, location: str, service: str,) -> str:
         """Returns a fully-qualified service string."""

diff --git a/google/cloud/dataproc_v1/types/__init__.py b/google/cloud/dataproc_v1/types/__init__.py
@@ -40,6 +40,7 @@
 from .clusters import (
     AcceleratorConfig,
     AutoscalingConfig,
+    AuxiliaryServicesConfig,
     Cluster,
     ClusterConfig,
     ClusterMetrics,
@@ -54,7 +55,6 @@
     EndpointConfig,
     GceClusterConfig,
     GetClusterRequest,
-    GkeClusterConfig,
     IdentityConfig,
     InstanceGroupConfig,
     KerberosConfig,
@@ -72,6 +72,7 @@
     StartClusterRequest,
     StopClusterRequest,
     UpdateClusterRequest,
+    VirtualClusterConfig,
 )
 from .jobs import (
     CancelJobRequest,
@@ -107,6 +108,11 @@
 from .shared import (
     EnvironmentConfig,
     ExecutionConfig,
+    GkeClusterConfig,
+    GkeNodePoolConfig,
+    GkeNodePoolTarget,
+    KubernetesClusterConfig,
+    KubernetesSoftwareConfig,
     PeripheralsConfig,
     RuntimeConfig,
     RuntimeInfo,
@@ -161,6 +167,7 @@
     "SparkSqlBatch",
     "AcceleratorConfig",
     "AutoscalingConfig",
+    "AuxiliaryServicesConfig",
     "Cluster",
     "ClusterConfig",
     "ClusterMetrics",
@@ -175,7 +182,6 @@
     "EndpointConfig",
     "GceClusterConfig",
     "GetClusterRequest",
-    "GkeClusterConfig",
     "IdentityConfig",
     "InstanceGroupConfig",
     "KerberosConfig",
@@ -193,6 +199,7 @@
     "StartClusterRequest",
     "StopClusterRequest",
     "UpdateClusterRequest",
+    "VirtualClusterConfig",
     "CancelJobRequest",
     "DeleteJobRequest",
     "GetJobRequest",
@@ -222,6 +229,11 @@
     "ClusterOperationStatus",
     "EnvironmentConfig",
     "ExecutionConfig",
+    "GkeClusterConfig",
+    "GkeNodePoolConfig",
+    "GkeNodePoolTarget",
+    "KubernetesClusterConfig",
+    "KubernetesSoftwareConfig",
     "PeripheralsConfig",
     "RuntimeConfig",
     "RuntimeInfo",