Skip to content
Permalink
Browse files
feat: add lifecycle config and reservation affinity support to v1 (vi…
…a synth) (#10)
  • Loading branch information
busunkim96 committed Mar 5, 2020
1 parent 6f5e0ed commit bb36194d4b0cfb6f2c5a0358625a17c629f71b21
Showing with 1,951 additions and 843 deletions.
  1. +6 −4 google/cloud/dataproc_v1/gapic/cluster_controller_client.py
  2. +19 −0 google/cloud/dataproc_v1/gapic/enums.py
  3. +1 −3 google/cloud/dataproc_v1/proto/autoscaling_policies.proto
  4. +79 −13 google/cloud/dataproc_v1/proto/clusters.proto
  5. +445 −77 google/cloud/dataproc_v1/proto/clusters_pb2.py
  6. +5 −5 google/cloud/dataproc_v1/proto/clusters_pb2_grpc.py
  7. +87 −15 google/cloud/dataproc_v1/proto/jobs.proto
  8. +688 −98 google/cloud/dataproc_v1/proto/jobs_pb2.py
  9. +2 −2 google/cloud/dataproc_v1/proto/jobs_pb2_grpc.py
  10. +1 −2 google/cloud/dataproc_v1/proto/operations.proto
  11. +1 −2 google/cloud/dataproc_v1/proto/shared.proto
  12. +6 −15 google/cloud/dataproc_v1/proto/workflow_templates.proto
  13. +0 −12 google/cloud/dataproc_v1/proto/workflow_templates_pb2.py
  14. +4 −4 google/cloud/dataproc_v1/proto/workflow_templates_pb2_grpc.py
  15. +1 −1 google/cloud/dataproc_v1beta2/gapic/autoscaling_policy_service_client.py
  16. +13 −11 google/cloud/dataproc_v1beta2/gapic/cluster_controller_client.py
  17. +2 −2 google/cloud/dataproc_v1beta2/gapic/enums.py
  18. +6 −6 google/cloud/dataproc_v1beta2/gapic/job_controller_client.py
  19. +1 −1 google/cloud/dataproc_v1beta2/gapic/transports/cluster_controller_grpc_transport.py
  20. +1 −1 google/cloud/dataproc_v1beta2/gapic/workflow_template_service_client.py
  21. +2 −3 google/cloud/dataproc_v1beta2/proto/autoscaling_policies.proto
  22. +2 −1 google/cloud/dataproc_v1beta2/proto/autoscaling_policies_pb2.py
  23. +2 −2 google/cloud/dataproc_v1beta2/proto/autoscaling_policies_pb2_grpc.py
  24. +60 −54 google/cloud/dataproc_v1beta2/proto/clusters.proto
  25. +144 −136 google/cloud/dataproc_v1beta2/proto/clusters_pb2.py
  26. +5 −5 google/cloud/dataproc_v1beta2/proto/clusters_pb2_grpc.py
  27. +93 −101 google/cloud/dataproc_v1beta2/proto/jobs.proto
  28. +247 −202 google/cloud/dataproc_v1beta2/proto/jobs_pb2.py
  29. +2 −2 google/cloud/dataproc_v1beta2/proto/jobs_pb2_grpc.py
  30. +1 −2 google/cloud/dataproc_v1beta2/proto/operations.proto
  31. +1 −2 google/cloud/dataproc_v1beta2/proto/shared.proto
  32. +9 −18 google/cloud/dataproc_v1beta2/proto/workflow_templates.proto
  33. +2 −14 google/cloud/dataproc_v1beta2/proto/workflow_templates_pb2.py
  34. +6 −6 google/cloud/dataproc_v1beta2/proto/workflow_templates_pb2_grpc.py
  35. +7 −21 synth.metadata
@@ -416,10 +416,12 @@ def update_cluster(
message :class:`~google.cloud.dataproc_v1.types.FieldMask`
graceful_decommission_timeout (Union[dict, ~google.cloud.dataproc_v1.types.Duration]): Optional. Timeout for graceful YARN decomissioning. Graceful
decommissioning allows removing nodes from the cluster without
interrupting jobs in progress. Timeout specifies how long to wait for jobs
in progress to finish before forcefully removing nodes (and potentially
interrupting jobs). Default timeout is 0 (for forceful decommission), and
the maximum allowed timeout is 1 day.
interrupting jobs in progress. Timeout specifies how long to wait for
jobs in progress to finish before forcefully removing nodes (and
potentially interrupting jobs). Default timeout is 0 (for forceful
decommission), and the maximum allowed timeout is 1 day. (see JSON
representation of
`Duration <https://developers.google.com/protocol-buffers/docs/proto3#json>`__).
Only supported on Dataproc image versions 1.2 and higher.
@@ -207,6 +207,25 @@ class Level(enum.IntEnum):
OFF = 8


class ReservationAffinity(object):
class Type(enum.IntEnum):
"""
Indicates whether to consume capacity from an reservation or not.
Attributes:
TYPE_UNSPECIFIED (int)
NO_RESERVATION (int): Do not consume from any allocated capacity.
ANY_RESERVATION (int): Consume any reservation available.
SPECIFIC_RESERVATION (int): Must consume from a specific reservation. Must specify key value fields
for specifying the reservations.
"""

TYPE_UNSPECIFIED = 0
NO_RESERVATION = 1
ANY_RESERVATION = 2
SPECIFIC_RESERVATION = 3


class WorkflowMetadata(object):
class State(enum.IntEnum):
"""
@@ -1,4 +1,4 @@
// Copyright 2019 Google LLC.
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

syntax = "proto3";

@@ -28,7 +27,6 @@ option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dat
option java_multiple_files = true;
option java_outer_classname = "AutoscalingPoliciesProto";
option java_package = "com.google.cloud.dataproc.v1";

option (google.api.resource_definition) = {
type: "dataproc.googleapis.com/Region"
pattern: "projects/{project}/regions/{region}"
@@ -1,4 +1,4 @@
// Copyright 2019 Google LLC.
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

syntax = "proto3";

@@ -20,7 +19,6 @@ package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/cloud/dataproc/v1/operations.proto";
import "google/cloud/dataproc/v1/shared.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/duration.proto";
@@ -40,7 +38,7 @@ service ClusterController {

// Creates a cluster in a project. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/clusters"
@@ -55,22 +53,22 @@ service ClusterController {

// Updates a cluster in a project. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
body: "cluster"
};
option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask";
option (google.longrunning.operation_info) = {
response_type: "Cluster"
metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
};
option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask";
}

// Deletes a cluster in a project. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
@@ -101,11 +99,11 @@ service ClusterController {

// Gets cluster diagnostic information. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// After the operation completes,
// [Operation.response][google.longrunning.Operation.response]
// contains
// [DiagnoseClusterResults](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
// [DiagnoseClusterResults](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose"
@@ -215,6 +213,9 @@ message ClusterConfig {

// Optional. Security settings for the cluster.
SecurityConfig security_config = 16 [(google.api.field_behavior) = OPTIONAL];

// Optional. Lifecycle setting for the cluster.
LifecycleConfig lifecycle_config = 17 [(google.api.field_behavior) = OPTIONAL];
}

// Autoscaling Policy config associated with the cluster.
@@ -322,9 +323,12 @@ message GceClusterConfig {
// [Project and instance
// metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
map<string, string> metadata = 5;

// Optional. Reservation Affinity for consuming Zonal reservation.
ReservationAffinity reservation_affinity = 11 [(google.api.field_behavior) = OPTIONAL];
}

// Optional. The config settings for Compute Engine resources in
// The config settings for Compute Engine resources in
// an instance group, such as a master or worker group.
message InstanceGroupConfig {
// Optional. The number of VM instances in the instance group.
@@ -438,7 +442,10 @@ message NodeInitializationAction {
string executable_file = 1 [(google.api.field_behavior) = REQUIRED];

// Optional. Amount of time executable has to complete. Default is
// 10 minutes. Cluster creation fails with an explanatory error message (the
// 10 minutes (see JSON representation of
// [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
//
// Cluster creation fails with an explanatory error message (the
// name of the executable that caused the error and the exceeded timeout
// period) if the executable is not completed at end of the timeout period.
google.protobuf.Duration execution_timeout = 2 [(google.api.field_behavior) = OPTIONAL];
@@ -495,7 +502,8 @@ message ClusterStatus {
(google.api.field_behavior) = OPTIONAL
];

// Output only. Time when this state was entered.
// Output only. Time when this state was entered (see JSON representation of
// [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. Additional state information that includes
@@ -613,6 +621,36 @@ message SoftwareConfig {
repeated Component optional_components = 3 [(google.api.field_behavior) = OPTIONAL];
}

// Specifies the cluster auto-delete schedule configuration.
message LifecycleConfig {
// Optional. The duration to keep the cluster alive while idling (when no jobs
// are running). Passing this threshold will cause the cluster to be
// deleted. Minimum value is 10 minutes; maximum value is 14 days (see JSON
// representation of
// [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json).
google.protobuf.Duration idle_delete_ttl = 1 [(google.api.field_behavior) = OPTIONAL];

// Either the exact time the cluster should be deleted at or
// the cluster maximum age.
oneof ttl {
// Optional. The time when cluster will be auto-deleted (see JSON representation of
// [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
google.protobuf.Timestamp auto_delete_time = 2 [(google.api.field_behavior) = OPTIONAL];

// Optional. The lifetime duration of cluster. The cluster will be
// auto-deleted at the end of this period. Minimum value is 10 minutes;
// maximum value is 14 days (see JSON representation of
// [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
google.protobuf.Duration auto_delete_ttl = 3 [(google.api.field_behavior) = OPTIONAL];
}

// Output only. The time when cluster became idle (most recent job finished)
// and became eligible for deletion due to idleness (see JSON representation
// of
// [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
google.protobuf.Timestamp idle_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Contains cluster daemon metrics, such as HDFS and YARN stats.
//
// **Beta Feature**: This report is available for testing purposes only. It may
@@ -671,7 +709,8 @@ message UpdateClusterRequest {
// interrupting jobs in progress. Timeout specifies how long to wait for jobs
// in progress to finish before forcefully removing nodes (and potentially
// interrupting jobs). Default timeout is 0 (for forceful decommission), and
// the maximum allowed timeout is 1 day.
// the maximum allowed timeout is 1 day. (see JSON representation of
// [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
//
// Only supported on Dataproc image versions 1.2 and higher.
google.protobuf.Duration graceful_decommission_timeout = 6 [(google.api.field_behavior) = OPTIONAL];
@@ -854,3 +893,30 @@ message DiagnoseClusterResults {
// diagnostics.
string output_uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Reservation Affinity for consuming Zonal reservation.
message ReservationAffinity {
// Indicates whether to consume capacity from an reservation or not.
enum Type {
TYPE_UNSPECIFIED = 0;

// Do not consume from any allocated capacity.
NO_RESERVATION = 1;

// Consume any reservation available.
ANY_RESERVATION = 2;

// Must consume from a specific reservation. Must specify key value fields
// for specifying the reservations.
SPECIFIC_RESERVATION = 3;
}

// Optional. Type of reservation to consume
Type consume_reservation_type = 1 [(google.api.field_behavior) = OPTIONAL];

// Optional. Corresponds to the label key of reservation resource.
string key = 2 [(google.api.field_behavior) = OPTIONAL];

// Optional. Corresponds to the label values of reservation resource.
repeated string values = 3 [(google.api.field_behavior) = OPTIONAL];
}

0 comments on commit bb36194

Please sign in to comment.