feat!: convert dataproc to typescript (#310)

* conversion to typescript * modified synth.py * fix samples test path issue * try to fix google.protobuf.Empty in docs * remove package chai from dependencies, Add source include protos for fixing google.protobuf.Empty * conversion to typescript * modified synth.py * fix samples test path issue * try to fix google.protobuf.Empty in docs * remove package chai from dependencies, Add source include protos for fixing google.protobuf.Empty * update latest published protos, gen docs for protos * lint fix * add protos into source in .jsdoc.js * remove unneeded AUTHOR and COPYING here * update upstream protos to fix [Empty](google.protobuf.Empty) * update package and jsdoc * fix merge Co-authored-by: Alexander Fenster <github@fenster.name>
googleapis · Feb 29, 2020 · 8170b0e · 8170b0e
1 parent 68cf3cd
commit 8170b0e
Show file tree

Hide file tree

Showing 100 changed files with 18,236 additions and 24,099 deletions.
diff --git a/packages/google-cloud-dataproc/.gitignore b/packages/google-cloud-dataproc/.gitignore
@@ -1,11 +1,14 @@
 **/*.log
 **/node_modules
 .coverage
+coverage
 .nyc_output
 docs/
+out/
 build/
+system-test/secrets.js
+system-test/*key.json
 *.lock
 .DS_Store
 package-lock.json
 __pycache__
-.idea
diff --git a/packages/google-cloud-dataproc/.jsdoc.js b/packages/google-cloud-dataproc/.jsdoc.js
@@ -31,7 +31,8 @@ module.exports = {
   source: {
     excludePattern: '(^|\\/|\\\\)[._]',
     include: [
-      'src'
+      'build/src',
+      'protos',
     ],
     includePattern: '\\.js$'
   },
@@ -42,7 +43,7 @@ module.exports = {
     systemName: '@google-cloud/dataproc',
     theme: 'lumen',
     default: {
-      "outputSourceFiles": false
+      outputSourceFiles: false
     }
   },
   markdown: {

diff --git a/packages/google-cloud-dataproc/README.md b/packages/google-cloud-dataproc/README.md
@@ -66,13 +66,13 @@ npm install @google-cloud/dataproc
 'use strict';
 
 function main(projectId, region, clusterName, jobFilePath) {
-  const dataproc = require('@google-cloud/dataproc').v1;
+  const dataproc = require('@google-cloud/dataproc');
   const {Storage} = require('@google-cloud/storage');
 
   const sleep = require('sleep');
 
   // Create a cluster client with the endpoint set to the desired cluster region
-  const clusterClient = new dataproc.ClusterControllerClient({
+  const clusterClient = new dataproc.v1.ClusterControllerClient({
     apiEndpoint: `${region}-dataproc.googleapis.com`,
   });
 

diff --git a/packages/google-cloud-dataproc/linkinator.config.json b/packages/google-cloud-dataproc/linkinator.config.json
@@ -3,7 +3,6 @@
   "skip": [
     "https://codecov.io/gh/googleapis/",
     "www.googleapis.com",
-    "createCluster.js",
     "img.shields.io"
   ]
 }
diff --git a/packages/google-cloud-dataproc/package.json b/packages/google-cloud-dataproc/package.json
@@ -5,15 +5,13 @@
   "license": "Apache-2.0",
   "author": "Google Inc",
   "engines": {
-    "node": ">=8.10.0"
+    "node": ">=8.13.0"
   },
   "repository": "googleapis/nodejs-dataproc",
-  "main": "src/index.js",
+  "main": "build/src/index.js",
   "files": [
-    "protos",
-    "src",
-    "AUTHORS",
-    "COPYING"
+    "build/protos",
+    "build/src"
   ],
   "keywords": [
     "google apis client",
@@ -29,32 +27,45 @@
     "Google Cloud Dataproc API"
   ],
   "scripts": {
-    "test": "c8 mocha",
-    "samples-test": "cd samples/ && npm link ../ && npm test && cd ../",
-    "lint": "eslint '**/*.js'",
+    "test": "c8 mocha build/test",
+    "samples-test": "cd samples/ && npm link ../ && npm install && npm test && cd ../",
+    "lint": "gts fix && eslint --fix samples/*.js",
     "docs": "jsdoc -c .jsdoc.js",
-    "system-test": "mocha system-test/*.js smoke-test/*.js --timeout 600000",
+    "system-test": "mocha build/system-test smoke-test/*.js",
     "fix": "eslint --fix '**/*.js'",
     "docs-test": "linkinator docs",
     "predocs-test": "npm run docs",
-    "prelint": "cd samples; npm link ../; npm i"
+    "prelint": "cd samples; npm link ../; npm i",
+    "clean": "gts clean",
+    "compile": "tsc -p . && cp -r protos build/",
+    "compile-protos": "compileProtos src",
+    "prepare": "npm run compile"
   },
   "dependencies": {
-    "google-gax": "^1.7.5",
+    "google-gax": "^1.14.1",
     "protobufjs": "^6.8.6"
   },
   "devDependencies": {
+    "@types/mocha": "^7.0.1",
+    "@types/node": "^12.0.0",
+    "c8": "^7.1.0",
     "codecov": "^3.0.2",
     "eslint": "^6.0.0",
     "eslint-config-prettier": "^6.0.0",
     "eslint-plugin-node": "^11.0.0",
     "eslint-plugin-prettier": "^3.0.0",
+    "gts": "^1.0.0",
     "jsdoc": "^3.6.2",
     "jsdoc-fresh": "^1.0.1",
     "jsdoc-region-tag": "^1.0.2",
-    "linkinator": "^2.0.0",
+    "linkinator": "^2.0.2",
     "mocha": "^7.0.0",
-    "c8": "^7.0.0",
-    "prettier": "^1.13.3"
+    "null-loader": "^3.0.0",
+    "pack-n-play": "^1.0.0-2",
+    "prettier": "^1.13.3",
+    "ts-loader": "^6.2.1",
+    "typescript": "^3.7.0",
+    "webpack": "^4.41.2",
+    "webpack-cli": "^3.3.10"
   }
 }
diff --git a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/autoscaling_policies.proto b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/autoscaling_policies.proto
@@ -1,4 +1,4 @@
-// Copyright 2019 Google LLC.
+// Copyright 2020 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 syntax = "proto3";
 
@@ -28,7 +27,6 @@ option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dat
 option java_multiple_files = true;
 option java_outer_classname = "AutoscalingPoliciesProto";
 option java_package = "com.google.cloud.dataproc.v1";
-
 option (google.api.resource_definition) = {
   type: "dataproc.googleapis.com/Region"
   pattern: "projects/{project}/regions/{region}"

diff --git a/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/clusters.proto b/packages/google-cloud-dataproc/protos/google/cloud/dataproc/v1/clusters.proto
@@ -1,4 +1,4 @@
-// Copyright 2019 Google LLC.
+// Copyright 2020 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-//
 
 syntax = "proto3";
 
@@ -20,7 +19,6 @@ package google.cloud.dataproc.v1;
 import "google/api/annotations.proto";
 import "google/api/client.proto";
 import "google/api/field_behavior.proto";
-import "google/cloud/dataproc/v1/operations.proto";
 import "google/cloud/dataproc/v1/shared.proto";
 import "google/longrunning/operations.proto";
 import "google/protobuf/duration.proto";
@@ -40,7 +38,7 @@ service ClusterController {
 
   // Creates a cluster in a project. The returned
   // [Operation.metadata][google.longrunning.Operation.metadata] will be
-  // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
+  // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
   rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) {
     option (google.api.http) = {
       post: "/v1/projects/{project_id}/regions/{region}/clusters"
@@ -55,22 +53,22 @@ service ClusterController {
 
   // Updates a cluster in a project. The returned
   // [Operation.metadata][google.longrunning.Operation.metadata] will be
-  // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
+  // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
   rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) {
     option (google.api.http) = {
       patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
       body: "cluster"
     };
+    option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask";
     option (google.longrunning.operation_info) = {
       response_type: "Cluster"
       metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata"
     };
-    option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask";
   }
 
   // Deletes a cluster in a project. The returned
   // [Operation.metadata][google.longrunning.Operation.metadata] will be
-  // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
+  // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
   rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) {
     option (google.api.http) = {
       delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}"
@@ -101,11 +99,11 @@ service ClusterController {
 
   // Gets cluster diagnostic information. The returned
   // [Operation.metadata][google.longrunning.Operation.metadata] will be
-  // [ClusterOperationMetadata](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
+  // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata).
   // After the operation completes,
   // [Operation.response][google.longrunning.Operation.response]
   // contains
-  // [DiagnoseClusterResults](/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
+  // [DiagnoseClusterResults](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults).
   rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) {
     option (google.api.http) = {
       post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose"
@@ -215,6 +213,9 @@ message ClusterConfig {
 
   // Optional. Security settings for the cluster.
   SecurityConfig security_config = 16 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Lifecycle setting for the cluster.
+  LifecycleConfig lifecycle_config = 17 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Autoscaling Policy config associated with the cluster.
@@ -322,9 +323,12 @@ message GceClusterConfig {
   // [Project and instance
   // metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)).
   map<string, string> metadata = 5;
+
+  // Optional. Reservation Affinity for consuming Zonal reservation.
+  ReservationAffinity reservation_affinity = 11 [(google.api.field_behavior) = OPTIONAL];
 }
 
-// Optional. The config settings for Compute Engine resources in
+// The config settings for Compute Engine resources in
 // an instance group, such as a master or worker group.
 message InstanceGroupConfig {
   // Optional. The number of VM instances in the instance group.
@@ -438,7 +442,10 @@ message NodeInitializationAction {
   string executable_file = 1 [(google.api.field_behavior) = REQUIRED];
 
   // Optional. Amount of time executable has to complete. Default is
-  // 10 minutes. Cluster creation fails with an explanatory error message (the
+  // 10 minutes (see JSON representation of
+  // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
+  //
+  // Cluster creation fails with an explanatory error message (the
   // name of the executable that caused the error and the exceeded timeout
   // period) if the executable is not completed at end of the timeout period.
   google.protobuf.Duration execution_timeout = 2 [(google.api.field_behavior) = OPTIONAL];
@@ -495,7 +502,8 @@ message ClusterStatus {
     (google.api.field_behavior) = OPTIONAL
   ];
 
-  // Output only. Time when this state was entered.
+  // Output only. Time when this state was entered (see JSON representation of
+  // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
   google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. Additional state information that includes
@@ -613,6 +621,36 @@ message SoftwareConfig {
   repeated Component optional_components = 3 [(google.api.field_behavior) = OPTIONAL];
 }
 
+// Specifies the cluster auto-delete schedule configuration.
+message LifecycleConfig {
+  // Optional. The duration to keep the cluster alive while idling (when no jobs
+  // are running). Passing this threshold will cause the cluster to be
+  // deleted. Minimum value is 10 minutes; maximum value is 14 days (see JSON
+  // representation of
+  // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json).
+  google.protobuf.Duration idle_delete_ttl = 1 [(google.api.field_behavior) = OPTIONAL];
+
+  // Either the exact time the cluster should be deleted at or
+  // the cluster maximum age.
+  oneof ttl {
+    // Optional. The time when cluster will be auto-deleted (see JSON representation of
+    // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
+    google.protobuf.Timestamp auto_delete_time = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The lifetime duration of cluster. The cluster will be
+    // auto-deleted at the end of this period. Minimum value is 10 minutes;
+    // maximum value is 14 days (see JSON representation of
+    // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
+    google.protobuf.Duration auto_delete_ttl = 3 [(google.api.field_behavior) = OPTIONAL];
+  }
+
+  // Output only. The time when cluster became idle (most recent job finished)
+  // and became eligible for deletion due to idleness (see JSON representation
+  // of
+  // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)).
+  google.protobuf.Timestamp idle_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
 // Contains cluster daemon metrics, such as HDFS and YARN stats.
 //
 // **Beta Feature**: This report is available for testing purposes only. It may
@@ -671,7 +709,8 @@ message UpdateClusterRequest {
   // interrupting jobs in progress. Timeout specifies how long to wait for jobs
   // in progress to finish before forcefully removing nodes (and potentially
   // interrupting jobs). Default timeout is 0 (for forceful decommission), and
-  // the maximum allowed timeout is 1 day.
+  // the maximum allowed timeout is 1 day. (see JSON representation of
+  // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)).
   //
   // Only supported on Dataproc image versions 1.2 and higher.
   google.protobuf.Duration graceful_decommission_timeout = 6 [(google.api.field_behavior) = OPTIONAL];
@@ -854,3 +893,30 @@ message DiagnoseClusterResults {
   // diagnostics.
   string output_uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
+
+// Reservation Affinity for consuming Zonal reservation.
+message ReservationAffinity {
+  // Indicates whether to consume capacity from an reservation or not.
+  enum Type {
+    TYPE_UNSPECIFIED = 0;
+
+    // Do not consume from any allocated capacity.
+    NO_RESERVATION = 1;
+
+    // Consume any reservation available.
+    ANY_RESERVATION = 2;
+
+    // Must consume from a specific reservation. Must specify key value fields
+    // for specifying the reservations.
+    SPECIFIC_RESERVATION = 3;
+  }
+
+  // Optional. Type of reservation to consume
+  Type consume_reservation_type = 1 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Corresponds to the label key of reservation resource.
+  string key = 2 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Corresponds to the label values of reservation resource.
+  repeated string values = 3 [(google.api.field_behavior) = OPTIONAL];
+}