feat: add DatasetVersion and dataset version RPCs to DatasetService

feat: add PersistentDiskSpec chore: remove backend configuration from the service config PiperOrigin-RevId: 570130129
googleapis · Oct 2, 2023 · 5813201 · 5813201
1 parent 4874ca8
commit 5813201
Show file tree

Hide file tree

Showing 12 changed files with 260 additions and 62 deletions.
diff --git a/google/cloud/aiplatform/v1/BUILD.bazel b/google/cloud/aiplatform/v1/BUILD.bazel
@@ -40,6 +40,7 @@ proto_library(
         "data_labeling_job.proto",
         "dataset.proto",
         "dataset_service.proto",
+        "dataset_version.proto",
         "deployed_index_ref.proto",
         "deployed_model_ref.proto",
         "encryption_spec.proto",

diff --git a/google/cloud/aiplatform/v1/aiplatform_v1.yaml b/google/cloud/aiplatform/v1/aiplatform_v1.yaml
@@ -140,57 +140,6 @@ documentation:
       permission-aware UIs and command-line tools, not for authorization
       checking. This operation may "fail open" without warning.
 
-backend:
-  rules:
-  - selector: 'google.cloud.aiplatform.v1.DatasetService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.EndpointService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.FeaturestoreOnlineServingService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.FeaturestoreService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.IndexEndpointService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.IndexService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.JobService.*'
-    deadline: 60.0
-  - selector: google.cloud.aiplatform.v1.MatchService.FindNeighbors
-    deadline: 30.0
-  - selector: google.cloud.aiplatform.v1.MatchService.ReadIndexDatapoints
-    deadline: 30.0
-  - selector: 'google.cloud.aiplatform.v1.MetadataService.*'
-    deadline: 60.0
-  - selector: google.cloud.aiplatform.v1.MigrationService.BatchMigrateResources
-    deadline: 60.0
-  - selector: google.cloud.aiplatform.v1.MigrationService.SearchMigratableResources
-    deadline: 60.0
-  - selector: google.cloud.aiplatform.v1.ModelGardenService.GetPublisherModel
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.ModelService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.PipelineService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.PredictionService.*'
-    deadline: 600.0
-  - selector: 'google.cloud.aiplatform.v1.ScheduleService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.SpecialistPoolService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.TensorboardService.*'
-    deadline: 60.0
-  - selector: 'google.cloud.aiplatform.v1.VizierService.*'
-    deadline: 60.0
-  - selector: google.cloud.location.Locations.GetLocation
-    deadline: 30.0
-  - selector: google.cloud.location.Locations.ListLocations
-    deadline: 30.0
-  - selector: 'google.iam.v1.IAMPolicy.*'
-    deadline: 60.0
-  - selector: 'google.longrunning.Operations.*'
-    deadline: 60.0
-
 http:
   rules:
   - selector: google.cloud.location.Locations.GetLocation
@@ -205,6 +154,7 @@ http:
     post: '/v1/{resource=projects/*/locations/*/featurestores/*}:getIamPolicy'
     additional_bindings:
     - post: '/v1/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:getIamPolicy'
+    - post: '/v1/{resource=projects/*/locations/*/notebookRuntimeTemplates/*}:getIamPolicy'
     - post: '/ui/{resource=projects/*/locations/*/featurestores/*}:getIamPolicy'
     - post: '/ui/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:getIamPolicy'
     - post: '/ui/{resource=projects/*/locations/*/models/*}:getIamPolicy'
@@ -216,6 +166,8 @@ http:
     additional_bindings:
     - post: '/v1/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:setIamPolicy'
       body: '*'
+    - post: '/v1/{resource=projects/*/locations/*/notebookRuntimeTemplates/*}:setIamPolicy'
+      body: '*'
     - post: '/ui/{resource=projects/*/locations/*/featurestores/*}:setIamPolicy'
       body: '*'
     - post: '/ui/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:setIamPolicy'
@@ -230,6 +182,7 @@ http:
     post: '/v1/{resource=projects/*/locations/*/featurestores/*}:testIamPermissions'
     additional_bindings:
     - post: '/v1/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:testIamPermissions'
+    - post: '/v1/{resource=projects/*/locations/*/notebookRuntimeTemplates/*}:testIamPermissions'
     - post: '/ui/{resource=projects/*/locations/*/featurestores/*}:testIamPermissions'
     - post: '/ui/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:testIamPermissions'
     - post: '/ui/{resource=projects/*/locations/*/models/*}:testIamPermissions'

diff --git a/google/cloud/aiplatform/v1/dataset_service.proto b/google/cloud/aiplatform/v1/dataset_service.proto
@@ -24,6 +24,7 @@ import "google/cloud/aiplatform/v1/annotation.proto";
 import "google/cloud/aiplatform/v1/annotation_spec.proto";
 import "google/cloud/aiplatform/v1/data_item.proto";
 import "google/cloud/aiplatform/v1/dataset.proto";
+import "google/cloud/aiplatform/v1/dataset_version.proto";
 import "google/cloud/aiplatform/v1/operation.proto";
 import "google/cloud/aiplatform/v1/saved_query.proto";
 import "google/longrunning/operations.proto";
@@ -122,6 +123,63 @@ service DatasetService {
     };
   }
 
+  // Create a version from a Dataset.
+  rpc CreateDatasetVersion(CreateDatasetVersionRequest)
+      returns (google.longrunning.Operation) {
+    option (google.api.http) = {
+      post: "/v1/{parent=projects/*/locations/*/datasets/*}/datasetVersions"
+      body: "dataset_version"
+    };
+    option (google.api.method_signature) = "parent,dataset_version";
+    option (google.longrunning.operation_info) = {
+      response_type: "DatasetVersion"
+      metadata_type: "CreateDatasetVersionOperationMetadata"
+    };
+  }
+
+  // Deletes a Dataset version.
+  rpc DeleteDatasetVersion(DeleteDatasetVersionRequest)
+      returns (google.longrunning.Operation) {
+    option (google.api.http) = {
+      delete: "/v1/{name=projects/*/locations/*/datasets/*/datasetVersions/*}"
+    };
+    option (google.api.method_signature) = "name";
+    option (google.longrunning.operation_info) = {
+      response_type: "google.protobuf.Empty"
+      metadata_type: "DeleteOperationMetadata"
+    };
+  }
+
+  // Gets a Dataset version.
+  rpc GetDatasetVersion(GetDatasetVersionRequest) returns (DatasetVersion) {
+    option (google.api.http) = {
+      get: "/v1/{name=projects/*/locations/*/datasets/*/datasetVersions/*}"
+    };
+    option (google.api.method_signature) = "name";
+  }
+
+  // Lists DatasetVersions in a Dataset.
+  rpc ListDatasetVersions(ListDatasetVersionsRequest)
+      returns (ListDatasetVersionsResponse) {
+    option (google.api.http) = {
+      get: "/v1/{parent=projects/*/locations/*/datasets/*}/datasetVersions"
+    };
+    option (google.api.method_signature) = "parent";
+  }
+
+  // Restores a dataset version.
+  rpc RestoreDatasetVersion(RestoreDatasetVersionRequest)
+      returns (google.longrunning.Operation) {
+    option (google.api.http) = {
+      get: "/v1/{name=projects/*/locations/*/datasets/*/datasetVersions/*}:restore"
+    };
+    option (google.api.method_signature) = "name";
+    option (google.longrunning.operation_info) = {
+      response_type: "DatasetVersion"
+      metadata_type: "RestoreDatasetVersionOperationMetadata"
+    };
+  }
+
   // Lists DataItems in a Dataset.
   rpc ListDataItems(ListDataItemsRequest) returns (ListDataItemsResponse) {
     option (google.api.http) = {
@@ -369,13 +427,118 @@ message ExportDataOperationMetadata {
   string gcs_output_directory = 2;
 }
 
+// Request message for
+// [DatasetService.CreateDatasetVersion][google.cloud.aiplatform.v1.DatasetService.CreateDatasetVersion].
+message CreateDatasetVersionRequest {
+  // Required. The name of the Dataset resource.
+  // Format:
+  // `projects/{project}/locations/{location}/datasets/{dataset}`
+  string parent = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/Dataset"
+    }
+  ];
+
+  // Required. The version to be created. The same CMEK policies with the
+  // original Dataset will be applied the dataset version. So here we don't need
+  // to specify the EncryptionSpecType here.
+  DatasetVersion dataset_version = 2 [(google.api.field_behavior) = REQUIRED];
+}
+
 // Runtime operation information for
 // [DatasetService.CreateDatasetVersion][google.cloud.aiplatform.v1.DatasetService.CreateDatasetVersion].
 message CreateDatasetVersionOperationMetadata {
   // The common part of the operation metadata.
   GenericOperationMetadata generic_metadata = 1;
 }
 
+// Request message for
+// [DatasetService.DeleteDatasetVersion][google.cloud.aiplatform.v1.DatasetService.DeleteDatasetVersion].
+message DeleteDatasetVersionRequest {
+  // Required. The resource name of the Dataset version to delete.
+  // Format:
+  // `projects/{project}/locations/{location}/datasets/{dataset}/datasetVersions/{dataset_version}`
+  string name = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/DatasetVersion"
+    }
+  ];
+}
+
+// Request message for
+// [DatasetService.GetDatasetVersion][google.cloud.aiplatform.v1.DatasetService.GetDatasetVersion].
+message GetDatasetVersionRequest {
+  // Required. The resource name of the Dataset version to delete.
+  // Format:
+  // `projects/{project}/locations/{location}/datasets/{dataset}/datasetVersions/{dataset_version}`
+  string name = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/DatasetVersion"
+    }
+  ];
+
+  // Mask specifying which fields to read.
+  google.protobuf.FieldMask read_mask = 2;
+}
+
+// Request message for
+// [DatasetService.ListDatasetVersions][google.cloud.aiplatform.v1.DatasetService.ListDatasetVersions].
+message ListDatasetVersionsRequest {
+  // Required. The resource name of the Dataset to list DatasetVersions from.
+  // Format:
+  // `projects/{project}/locations/{location}/datasets/{dataset}`
+  string parent = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/Dataset"
+    }
+  ];
+
+  // Optional. The standard list filter.
+  string filter = 2 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. The standard list page size.
+  int32 page_size = 3 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. The standard list page token.
+  string page_token = 4 [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Mask specifying which fields to read.
+  google.protobuf.FieldMask read_mask = 5
+      [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. A comma-separated list of fields to order by, sorted in ascending
+  // order. Use "desc" after a field name for descending.
+  string order_by = 6 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// Response message for
+// [DatasetService.ListDatasetVersions][google.cloud.aiplatform.v1.DatasetService.ListDatasetVersions].
+message ListDatasetVersionsResponse {
+  // A list of DatasetVersions that matches the specified filter in the request.
+  repeated DatasetVersion dataset_versions = 1;
+
+  // The standard List next-page token.
+  string next_page_token = 2;
+}
+
+// Request message for
+// [DatasetService.RestoreDatasetVersion][google.cloud.aiplatform.v1.DatasetService.RestoreDatasetVersion].
+message RestoreDatasetVersionRequest {
+  // Required. The name of the DatasetVersion resource.
+  // Format:
+  // `projects/{project}/locations/{location}/datasets/{dataset}/datasetVersions/{dataset_version}`
+  string name = 1 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.resource_reference) = {
+      type: "aiplatform.googleapis.com/DatasetVersion"
+    }
+  ];
+}
+
 // Runtime operation information for
 // [DatasetService.RestoreDatasetVersion][google.cloud.aiplatform.v1.DatasetService.RestoreDatasetVersion].
 message RestoreDatasetVersionOperationMetadata {

diff --git a/google/cloud/aiplatform/v1/dataset_version.proto b/google/cloud/aiplatform/v1/dataset_version.proto
@@ -0,0 +1,55 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.cloud.aiplatform.v1;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+import "google/protobuf/timestamp.proto";
+
+option csharp_namespace = "Google.Cloud.AIPlatform.V1";
+option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb";
+option java_multiple_files = true;
+option java_outer_classname = "DatasetVersionProto";
+option java_package = "com.google.cloud.aiplatform.v1";
+option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
+option ruby_package = "Google::Cloud::AIPlatform::V1";
+
+// Describes the dataset version.
+message DatasetVersion {
+  option (google.api.resource) = {
+    type: "aiplatform.googleapis.com/DatasetVersion"
+    pattern: "projects/{project}/locations/{location}/datasets/{dataset}/datasetVersions/{dataset_version}"
+  };
+
+  // Output only. The resource name of the DatasetVersion.
+  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when this DatasetVersion was created.
+  google.protobuf.Timestamp create_time = 2
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Timestamp when this DatasetVersion was last updated.
+  google.protobuf.Timestamp update_time = 6
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Used to perform consistent read-modify-write updates. If not set, a blind
+  // "overwrite" update happens.
+  string etag = 3;
+
+  // Output only. Name of the associated BigQuery dataset.
+  string big_query_dataset_name = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
diff --git a/google/cloud/aiplatform/v1/endpoint.proto b/google/cloud/aiplatform/v1/endpoint.proto
@@ -164,7 +164,7 @@ message DeployedModel {
   // Immutable. The ID of the DeployedModel. If not provided upon deployment,
   // Vertex AI will generate a value for this ID.
   //
-  // This value should be 1-10 characters, and valid characters are /[0-9]/.
+  // This value should be 1-10 characters, and valid characters are `/[0-9]/`.
   string id = 1 [(google.api.field_behavior) = IMMUTABLE];
 
   // Required. The resource name of the Model that this is the deployment of.

diff --git a/google/cloud/aiplatform/v1/index.proto b/google/cloud/aiplatform/v1/index.proto
@@ -45,7 +45,7 @@ message Index {
     INDEX_UPDATE_METHOD_UNSPECIFIED = 0;
 
     // BatchUpdate: user can call UpdateIndex with files on Cloud Storage of
-    // datapoints to update.
+    // Datapoints to update.
     BATCH_UPDATE = 1;
 
     // StreamUpdate: user can call UpsertDatapoints/DeleteDatapoints to update
@@ -132,13 +132,13 @@ message IndexDatapoint {
   // Restriction of a datapoint which describe its attributes(tokens) from each
   // of several attribute categories(namespaces).
   message Restriction {
-    // The namespace of this restriction. eg: color.
+    // The namespace of this restriction. e.g.: color.
     string namespace = 1;
 
-    // The attributes to allow in this namespace. eg: 'red'
+    // The attributes to allow in this namespace. e.g.: 'red'
     repeated string allow_list = 2;
 
-    // The attributes to deny in this namespace. eg: 'blue'
+    // The attributes to deny in this namespace. e.g.: 'blue'
     repeated string deny_list = 3;
   }
 
@@ -163,7 +163,7 @@ message IndexDatapoint {
 
   // Optional. List of Restrict of the datapoint, used to perform "restricted
   // searches" where boolean rule are used to filter the subset of the database
-  // eligible for matching. See:
+  // eligible for matching. This uses categorical tokens. See:
   // https://cloud.google.com/vertex-ai/docs/matching-engine/filtering
   repeated Restriction restricts = 4 [(google.api.field_behavior) = OPTIONAL];
 

diff --git a/google/cloud/aiplatform/v1/index_endpoint.proto b/google/cloud/aiplatform/v1/index_endpoint.proto
@@ -233,7 +233,10 @@ message DeployedIndex {
   //
   // The value should be the name of the address
   // (https://cloud.google.com/compute/docs/reference/rest/v1/addresses)
-  // Example: 'vertex-ai-ip-range'.
+  // Example: ['vertex-ai-ip-range'].
+  //
+  // For more information about subnets and network IP ranges, please see
+  // https://cloud.google.com/vpc/docs/subnets#manually_created_subnet_ip_ranges.
   repeated string reserved_ip_ranges = 10
       [(google.api.field_behavior) = OPTIONAL];