Skip to content
This repository has been archived by the owner on Nov 29, 2023. It is now read-only.

feat: add lifecycle config and reservation affinity support to v1 (via synth) #10

Merged
merged 1 commit into from Mar 5, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 6 additions & 4 deletions google/cloud/dataproc_v1/gapic/cluster_controller_client.py
Expand Up @@ -416,10 +416,12 @@ def update_cluster(
message :class:`~google.cloud.dataproc_v1.types.FieldMask`
graceful_decommission_timeout (Union[dict, ~google.cloud.dataproc_v1.types.Duration]): Optional. Timeout for graceful YARN decomissioning. Graceful
decommissioning allows removing nodes from the cluster without
interrupting jobs in progress. Timeout specifies how long to wait for jobs
in progress to finish before forcefully removing nodes (and potentially
interrupting jobs). Default timeout is 0 (for forceful decommission), and
the maximum allowed timeout is 1 day.
interrupting jobs in progress. Timeout specifies how long to wait for
jobs in progress to finish before forcefully removing nodes (and
potentially interrupting jobs). Default timeout is 0 (for forceful
decommission), and the maximum allowed timeout is 1 day. (see JSON
representation of
`Duration <https://developers.google.com/protocol-buffers/docs/proto3#json>`__).

Only supported on Dataproc image versions 1.2 and higher.

Expand Down
19 changes: 19 additions & 0 deletions google/cloud/dataproc_v1/gapic/enums.py
Expand Up @@ -207,6 +207,25 @@ class Level(enum.IntEnum):
OFF = 8


class ReservationAffinity(object):
class Type(enum.IntEnum):
"""
Indicates whether to consume capacity from an reservation or not.

Attributes:
TYPE_UNSPECIFIED (int)
NO_RESERVATION (int): Do not consume from any allocated capacity.
ANY_RESERVATION (int): Consume any reservation available.
SPECIFIC_RESERVATION (int): Must consume from a specific reservation. Must specify key value fields
for specifying the reservations.
"""

TYPE_UNSPECIFIED = 0
NO_RESERVATION = 1
ANY_RESERVATION = 2
SPECIFIC_RESERVATION = 3


class WorkflowMetadata(object):
class State(enum.IntEnum):
"""
Expand Down
4 changes: 1 addition & 3 deletions google/cloud/dataproc_v1/proto/autoscaling_policies.proto
@@ -1,4 +1,4 @@
// Copyright 2019 Google LLC.
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -11,7 +11,6 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

syntax = "proto3";

Expand All @@ -28,7 +27,6 @@ option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dat
option java_multiple_files = true;
option java_outer_classname = "AutoscalingPoliciesProto";
option java_package = "com.google.cloud.dataproc.v1";

option (google.api.resource_definition) = {
type: "dataproc.googleapis.com/Region"
pattern: "projects/{project}/regions/{region}"
Expand Down
92 changes: 79 additions & 13 deletions google/cloud/dataproc_v1/proto/clusters.proto
@@ -1,4 +1,4 @@
// Copyright 2019 Google LLC.
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -11,7 +11,6 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

syntax = "proto3";

Expand All @@ -20,7 +19,6 @@ package google.cloud.dataproc.v1;
import "google/api/annotations.proto";
import "google/api/client.proto";
import "google/api/field_behavior.proto";
import "google/cloud/dataproc/v1/operations.proto";
import "google/cloud/dataproc/v1/shared.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/duration.proto";
Expand All @@ -40,7 +38,7 @@ service ClusterController {

// Creates a cluster in a project. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/clusters"
Expand All @@ -55,22 +53,22 @@ service ClusterController {

// Updates a cluster in a project. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
body: "cluster"
};
option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask";
option (google.longrunning.operation_info) = {
response_type: "Cluster"
metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
};
option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask";
}

// Deletes a cluster in a project. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
Expand Down Expand Up @@ -101,11 +99,11 @@ service ClusterController {

// Gets cluster diagnostic information. The returned
// [Operation.metadata][google.longrunning.Operation.metadata] will be
// [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
// After the operation completes,
// [Operation.response][google.longrunning.Operation.response]
// contains
// [DiagnoseClusterResults](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
// [DiagnoseClusterResults](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) {
option (google.api.http) = {
post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose"
Expand Down Expand Up @@ -215,6 +213,9 @@ message ClusterConfig {

// Optional. Security settings for the cluster.
SecurityConfig security_config = 16 [(google.api.field_behavior) = OPTIONAL];

// Optional. Lifecycle setting for the cluster.
LifecycleConfig lifecycle_config = 17 [(google.api.field_behavior) = OPTIONAL];
}

// Autoscaling Policy config associated with the cluster.
Expand Down Expand Up @@ -322,9 +323,12 @@ message GceClusterConfig {
// [Project and instance
// metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
map<string, string> metadata = 5;

// Optional. Reservation Affinity for consuming Zonal reservation.
ReservationAffinity reservation_affinity = 11 [(google.api.field_behavior) = OPTIONAL];
}

// Optional. The config settings for Compute Engine resources in
// The config settings for Compute Engine resources in
// an instance group, such as a master or worker group.
message InstanceGroupConfig {
// Optional. The number of VM instances in the instance group.
Expand Down Expand Up @@ -438,7 +442,10 @@ message NodeInitializationAction {
string executable_file = 1 [(google.api.field_behavior) = REQUIRED];

// Optional. Amount of time executable has to complete. Default is
// 10 minutes. Cluster creation fails with an explanatory error message (the
// 10 minutes (see JSON representation of
// [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
//
// Cluster creation fails with an explanatory error message (the
// name of the executable that caused the error and the exceeded timeout
// period) if the executable is not completed at end of the timeout period.
google.protobuf.Duration execution_timeout = 2 [(google.api.field_behavior) = OPTIONAL];
Expand Down Expand Up @@ -495,7 +502,8 @@ message ClusterStatus {
(google.api.field_behavior) = OPTIONAL
];

// Output only. Time when this state was entered.
// Output only. Time when this state was entered (see JSON representation of
// [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. Additional state information that includes
Expand Down Expand Up @@ -613,6 +621,36 @@ message SoftwareConfig {
repeated Component optional_components = 3 [(google.api.field_behavior) = OPTIONAL];
}

// Specifies the cluster auto-delete schedule configuration.
message LifecycleConfig {
// Optional. The duration to keep the cluster alive while idling (when no jobs
// are running). Passing this threshold will cause the cluster to be
// deleted. Minimum value is 10 minutes; maximum value is 14 days (see JSON
// representation of
// [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json).
google.protobuf.Duration idle_delete_ttl = 1 [(google.api.field_behavior) = OPTIONAL];

// Either the exact time the cluster should be deleted at or
// the cluster maximum age.
oneof ttl {
// Optional. The time when cluster will be auto-deleted (see JSON representation of
// [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
google.protobuf.Timestamp auto_delete_time = 2 [(google.api.field_behavior) = OPTIONAL];

// Optional. The lifetime duration of cluster. The cluster will be
// auto-deleted at the end of this period. Minimum value is 10 minutes;
// maximum value is 14 days (see JSON representation of
// [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
google.protobuf.Duration auto_delete_ttl = 3 [(google.api.field_behavior) = OPTIONAL];
}

// Output only. The time when cluster became idle (most recent job finished)
// and became eligible for deletion due to idleness (see JSON representation
// of
// [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
google.protobuf.Timestamp idle_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Contains cluster daemon metrics, such as HDFS and YARN stats.
//
// **Beta Feature**: This report is available for testing purposes only. It may
Expand Down Expand Up @@ -671,7 +709,8 @@ message UpdateClusterRequest {
// interrupting jobs in progress. Timeout specifies how long to wait for jobs
// in progress to finish before forcefully removing nodes (and potentially
// interrupting jobs). Default timeout is 0 (for forceful decommission), and
// the maximum allowed timeout is 1 day.
// the maximum allowed timeout is 1 day. (see JSON representation of
// [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
//
// Only supported on Dataproc image versions 1.2 and higher.
google.protobuf.Duration graceful_decommission_timeout = 6 [(google.api.field_behavior) = OPTIONAL];
Expand Down Expand Up @@ -854,3 +893,30 @@ message DiagnoseClusterResults {
// diagnostics.
string output_uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Reservation Affinity for consuming Zonal reservation.
message ReservationAffinity {
// Indicates whether to consume capacity from an reservation or not.
enum Type {
TYPE_UNSPECIFIED = 0;

// Do not consume from any allocated capacity.
NO_RESERVATION = 1;

// Consume any reservation available.
ANY_RESERVATION = 2;

// Must consume from a specific reservation. Must specify key value fields
// for specifying the reservations.
SPECIFIC_RESERVATION = 3;
}

// Optional. Type of reservation to consume
Type consume_reservation_type = 1 [(google.api.field_behavior) = OPTIONAL];

// Optional. Corresponds to the label key of reservation resource.
string key = 2 [(google.api.field_behavior) = OPTIONAL];

// Optional. Corresponds to the label values of reservation resource.
repeated string values = 3 [(google.api.field_behavior) = OPTIONAL];
}