From 581320192968f4087c00fb306fb1d47c2cb00547 Mon Sep 17 00:00:00 2001 From: Google APIs Date: Mon, 2 Oct 2023 11:46:39 -0700 Subject: [PATCH] feat: add DatasetVersion and dataset version RPCs to DatasetService feat: add PersistentDiskSpec chore: remove backend configuration from the service config PiperOrigin-RevId: 570130129 --- google/cloud/aiplatform/v1/BUILD.bazel | 1 + google/cloud/aiplatform/v1/aiplatform_v1.yaml | 55 +----- .../cloud/aiplatform/v1/dataset_service.proto | 163 ++++++++++++++++++ .../cloud/aiplatform/v1/dataset_version.proto | 55 ++++++ google/cloud/aiplatform/v1/endpoint.proto | 2 +- google/cloud/aiplatform/v1/index.proto | 10 +- .../cloud/aiplatform/v1/index_endpoint.proto | 5 +- .../aiplatform/v1/machine_resources.proto | 14 ++ .../aiplatform/v1/model_monitoring.proto | 7 + google/cloud/aiplatform/v1/pipeline_job.proto | 4 +- .../aiplatform/v1/pipeline_service.proto | 2 +- .../aiplatform/v1/tensorboard_service.proto | 4 +- 12 files changed, 260 insertions(+), 62 deletions(-) create mode 100644 google/cloud/aiplatform/v1/dataset_version.proto diff --git a/google/cloud/aiplatform/v1/BUILD.bazel b/google/cloud/aiplatform/v1/BUILD.bazel index 38d17ecc88848..e6bd6351f4484 100644 --- a/google/cloud/aiplatform/v1/BUILD.bazel +++ b/google/cloud/aiplatform/v1/BUILD.bazel @@ -40,6 +40,7 @@ proto_library( "data_labeling_job.proto", "dataset.proto", "dataset_service.proto", + "dataset_version.proto", "deployed_index_ref.proto", "deployed_model_ref.proto", "encryption_spec.proto", diff --git a/google/cloud/aiplatform/v1/aiplatform_v1.yaml b/google/cloud/aiplatform/v1/aiplatform_v1.yaml index 7741e07851571..1b8bcd148887a 100644 --- a/google/cloud/aiplatform/v1/aiplatform_v1.yaml +++ b/google/cloud/aiplatform/v1/aiplatform_v1.yaml @@ -140,57 +140,6 @@ documentation: permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. -backend: - rules: - - selector: 'google.cloud.aiplatform.v1.DatasetService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.EndpointService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.FeaturestoreOnlineServingService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.FeaturestoreService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.IndexEndpointService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.IndexService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.JobService.*' - deadline: 60.0 - - selector: google.cloud.aiplatform.v1.MatchService.FindNeighbors - deadline: 30.0 - - selector: google.cloud.aiplatform.v1.MatchService.ReadIndexDatapoints - deadline: 30.0 - - selector: 'google.cloud.aiplatform.v1.MetadataService.*' - deadline: 60.0 - - selector: google.cloud.aiplatform.v1.MigrationService.BatchMigrateResources - deadline: 60.0 - - selector: google.cloud.aiplatform.v1.MigrationService.SearchMigratableResources - deadline: 60.0 - - selector: google.cloud.aiplatform.v1.ModelGardenService.GetPublisherModel - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.ModelService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.PipelineService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.PredictionService.*' - deadline: 600.0 - - selector: 'google.cloud.aiplatform.v1.ScheduleService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.SpecialistPoolService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.TensorboardService.*' - deadline: 60.0 - - selector: 'google.cloud.aiplatform.v1.VizierService.*' - deadline: 60.0 - - selector: google.cloud.location.Locations.GetLocation - deadline: 30.0 - - selector: google.cloud.location.Locations.ListLocations - deadline: 30.0 - - selector: 'google.iam.v1.IAMPolicy.*' - deadline: 60.0 - - selector: 'google.longrunning.Operations.*' - deadline: 60.0 - http: rules: - selector: google.cloud.location.Locations.GetLocation @@ -205,6 +154,7 @@ http: post: '/v1/{resource=projects/*/locations/*/featurestores/*}:getIamPolicy' additional_bindings: - post: '/v1/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:getIamPolicy' + - post: '/v1/{resource=projects/*/locations/*/notebookRuntimeTemplates/*}:getIamPolicy' - post: '/ui/{resource=projects/*/locations/*/featurestores/*}:getIamPolicy' - post: '/ui/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:getIamPolicy' - post: '/ui/{resource=projects/*/locations/*/models/*}:getIamPolicy' @@ -216,6 +166,8 @@ http: additional_bindings: - post: '/v1/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:setIamPolicy' body: '*' + - post: '/v1/{resource=projects/*/locations/*/notebookRuntimeTemplates/*}:setIamPolicy' + body: '*' - post: '/ui/{resource=projects/*/locations/*/featurestores/*}:setIamPolicy' body: '*' - post: '/ui/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:setIamPolicy' @@ -230,6 +182,7 @@ http: post: '/v1/{resource=projects/*/locations/*/featurestores/*}:testIamPermissions' additional_bindings: - post: '/v1/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:testIamPermissions' + - post: '/v1/{resource=projects/*/locations/*/notebookRuntimeTemplates/*}:testIamPermissions' - post: '/ui/{resource=projects/*/locations/*/featurestores/*}:testIamPermissions' - post: '/ui/{resource=projects/*/locations/*/featurestores/*/entityTypes/*}:testIamPermissions' - post: '/ui/{resource=projects/*/locations/*/models/*}:testIamPermissions' diff --git a/google/cloud/aiplatform/v1/dataset_service.proto b/google/cloud/aiplatform/v1/dataset_service.proto index 5968668aa46ba..a9186a335d881 100644 --- a/google/cloud/aiplatform/v1/dataset_service.proto +++ b/google/cloud/aiplatform/v1/dataset_service.proto @@ -24,6 +24,7 @@ import "google/cloud/aiplatform/v1/annotation.proto"; import "google/cloud/aiplatform/v1/annotation_spec.proto"; import "google/cloud/aiplatform/v1/data_item.proto"; import "google/cloud/aiplatform/v1/dataset.proto"; +import "google/cloud/aiplatform/v1/dataset_version.proto"; import "google/cloud/aiplatform/v1/operation.proto"; import "google/cloud/aiplatform/v1/saved_query.proto"; import "google/longrunning/operations.proto"; @@ -122,6 +123,63 @@ service DatasetService { }; } + // Create a version from a Dataset. + rpc CreateDatasetVersion(CreateDatasetVersionRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + post: "/v1/{parent=projects/*/locations/*/datasets/*}/datasetVersions" + body: "dataset_version" + }; + option (google.api.method_signature) = "parent,dataset_version"; + option (google.longrunning.operation_info) = { + response_type: "DatasetVersion" + metadata_type: "CreateDatasetVersionOperationMetadata" + }; + } + + // Deletes a Dataset version. + rpc DeleteDatasetVersion(DeleteDatasetVersionRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + delete: "/v1/{name=projects/*/locations/*/datasets/*/datasetVersions/*}" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "DeleteOperationMetadata" + }; + } + + // Gets a Dataset version. + rpc GetDatasetVersion(GetDatasetVersionRequest) returns (DatasetVersion) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/datasets/*/datasetVersions/*}" + }; + option (google.api.method_signature) = "name"; + } + + // Lists DatasetVersions in a Dataset. + rpc ListDatasetVersions(ListDatasetVersionsRequest) + returns (ListDatasetVersionsResponse) { + option (google.api.http) = { + get: "/v1/{parent=projects/*/locations/*/datasets/*}/datasetVersions" + }; + option (google.api.method_signature) = "parent"; + } + + // Restores a dataset version. + rpc RestoreDatasetVersion(RestoreDatasetVersionRequest) + returns (google.longrunning.Operation) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/datasets/*/datasetVersions/*}:restore" + }; + option (google.api.method_signature) = "name"; + option (google.longrunning.operation_info) = { + response_type: "DatasetVersion" + metadata_type: "RestoreDatasetVersionOperationMetadata" + }; + } + // Lists DataItems in a Dataset. rpc ListDataItems(ListDataItemsRequest) returns (ListDataItemsResponse) { option (google.api.http) = { @@ -369,6 +427,25 @@ message ExportDataOperationMetadata { string gcs_output_directory = 2; } +// Request message for +// [DatasetService.CreateDatasetVersion][google.cloud.aiplatform.v1.DatasetService.CreateDatasetVersion]. +message CreateDatasetVersionRequest { + // Required. The name of the Dataset resource. + // Format: + // `projects/{project}/locations/{location}/datasets/{dataset}` + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "aiplatform.googleapis.com/Dataset" + } + ]; + + // Required. The version to be created. The same CMEK policies with the + // original Dataset will be applied the dataset version. So here we don't need + // to specify the EncryptionSpecType here. + DatasetVersion dataset_version = 2 [(google.api.field_behavior) = REQUIRED]; +} + // Runtime operation information for // [DatasetService.CreateDatasetVersion][google.cloud.aiplatform.v1.DatasetService.CreateDatasetVersion]. message CreateDatasetVersionOperationMetadata { @@ -376,6 +453,92 @@ message CreateDatasetVersionOperationMetadata { GenericOperationMetadata generic_metadata = 1; } +// Request message for +// [DatasetService.DeleteDatasetVersion][google.cloud.aiplatform.v1.DatasetService.DeleteDatasetVersion]. +message DeleteDatasetVersionRequest { + // Required. The resource name of the Dataset version to delete. + // Format: + // `projects/{project}/locations/{location}/datasets/{dataset}/datasetVersions/{dataset_version}` + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "aiplatform.googleapis.com/DatasetVersion" + } + ]; +} + +// Request message for +// [DatasetService.GetDatasetVersion][google.cloud.aiplatform.v1.DatasetService.GetDatasetVersion]. +message GetDatasetVersionRequest { + // Required. The resource name of the Dataset version to delete. + // Format: + // `projects/{project}/locations/{location}/datasets/{dataset}/datasetVersions/{dataset_version}` + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "aiplatform.googleapis.com/DatasetVersion" + } + ]; + + // Mask specifying which fields to read. + google.protobuf.FieldMask read_mask = 2; +} + +// Request message for +// [DatasetService.ListDatasetVersions][google.cloud.aiplatform.v1.DatasetService.ListDatasetVersions]. +message ListDatasetVersionsRequest { + // Required. The resource name of the Dataset to list DatasetVersions from. + // Format: + // `projects/{project}/locations/{location}/datasets/{dataset}` + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "aiplatform.googleapis.com/Dataset" + } + ]; + + // Optional. The standard list filter. + string filter = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The standard list page size. + int32 page_size = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. The standard list page token. + string page_token = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Mask specifying which fields to read. + google.protobuf.FieldMask read_mask = 5 + [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A comma-separated list of fields to order by, sorted in ascending + // order. Use "desc" after a field name for descending. + string order_by = 6 [(google.api.field_behavior) = OPTIONAL]; +} + +// Response message for +// [DatasetService.ListDatasetVersions][google.cloud.aiplatform.v1.DatasetService.ListDatasetVersions]. +message ListDatasetVersionsResponse { + // A list of DatasetVersions that matches the specified filter in the request. + repeated DatasetVersion dataset_versions = 1; + + // The standard List next-page token. + string next_page_token = 2; +} + +// Request message for +// [DatasetService.RestoreDatasetVersion][google.cloud.aiplatform.v1.DatasetService.RestoreDatasetVersion]. +message RestoreDatasetVersionRequest { + // Required. The name of the DatasetVersion resource. + // Format: + // `projects/{project}/locations/{location}/datasets/{dataset}/datasetVersions/{dataset_version}` + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "aiplatform.googleapis.com/DatasetVersion" + } + ]; +} + // Runtime operation information for // [DatasetService.RestoreDatasetVersion][google.cloud.aiplatform.v1.DatasetService.RestoreDatasetVersion]. message RestoreDatasetVersionOperationMetadata { diff --git a/google/cloud/aiplatform/v1/dataset_version.proto b/google/cloud/aiplatform/v1/dataset_version.proto new file mode 100644 index 0000000000000..5a6dd1110d345 --- /dev/null +++ b/google/cloud/aiplatform/v1/dataset_version.proto @@ -0,0 +1,55 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.cloud.aiplatform.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.AIPlatform.V1"; +option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb"; +option java_multiple_files = true; +option java_outer_classname = "DatasetVersionProto"; +option java_package = "com.google.cloud.aiplatform.v1"; +option php_namespace = "Google\\Cloud\\AIPlatform\\V1"; +option ruby_package = "Google::Cloud::AIPlatform::V1"; + +// Describes the dataset version. +message DatasetVersion { + option (google.api.resource) = { + type: "aiplatform.googleapis.com/DatasetVersion" + pattern: "projects/{project}/locations/{location}/datasets/{dataset}/datasetVersions/{dataset_version}" + }; + + // Output only. The resource name of the DatasetVersion. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Timestamp when this DatasetVersion was created. + google.protobuf.Timestamp create_time = 2 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Timestamp when this DatasetVersion was last updated. + google.protobuf.Timestamp update_time = 6 + [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Used to perform consistent read-modify-write updates. If not set, a blind + // "overwrite" update happens. + string etag = 3; + + // Output only. Name of the associated BigQuery dataset. + string big_query_dataset_name = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; +} diff --git a/google/cloud/aiplatform/v1/endpoint.proto b/google/cloud/aiplatform/v1/endpoint.proto index e014e10611f04..b29ec0c03054a 100644 --- a/google/cloud/aiplatform/v1/endpoint.proto +++ b/google/cloud/aiplatform/v1/endpoint.proto @@ -164,7 +164,7 @@ message DeployedModel { // Immutable. The ID of the DeployedModel. If not provided upon deployment, // Vertex AI will generate a value for this ID. // - // This value should be 1-10 characters, and valid characters are /[0-9]/. + // This value should be 1-10 characters, and valid characters are `/[0-9]/`. string id = 1 [(google.api.field_behavior) = IMMUTABLE]; // Required. The resource name of the Model that this is the deployment of. diff --git a/google/cloud/aiplatform/v1/index.proto b/google/cloud/aiplatform/v1/index.proto index 192a8d53bcf86..1dfaca9b8ac11 100644 --- a/google/cloud/aiplatform/v1/index.proto +++ b/google/cloud/aiplatform/v1/index.proto @@ -45,7 +45,7 @@ message Index { INDEX_UPDATE_METHOD_UNSPECIFIED = 0; // BatchUpdate: user can call UpdateIndex with files on Cloud Storage of - // datapoints to update. + // Datapoints to update. BATCH_UPDATE = 1; // StreamUpdate: user can call UpsertDatapoints/DeleteDatapoints to update @@ -132,13 +132,13 @@ message IndexDatapoint { // Restriction of a datapoint which describe its attributes(tokens) from each // of several attribute categories(namespaces). message Restriction { - // The namespace of this restriction. eg: color. + // The namespace of this restriction. e.g.: color. string namespace = 1; - // The attributes to allow in this namespace. eg: 'red' + // The attributes to allow in this namespace. e.g.: 'red' repeated string allow_list = 2; - // The attributes to deny in this namespace. eg: 'blue' + // The attributes to deny in this namespace. e.g.: 'blue' repeated string deny_list = 3; } @@ -163,7 +163,7 @@ message IndexDatapoint { // Optional. List of Restrict of the datapoint, used to perform "restricted // searches" where boolean rule are used to filter the subset of the database - // eligible for matching. See: + // eligible for matching. This uses categorical tokens. See: // https://cloud.google.com/vertex-ai/docs/matching-engine/filtering repeated Restriction restricts = 4 [(google.api.field_behavior) = OPTIONAL]; diff --git a/google/cloud/aiplatform/v1/index_endpoint.proto b/google/cloud/aiplatform/v1/index_endpoint.proto index 89672421d00b1..cca3fc3d5dd5c 100644 --- a/google/cloud/aiplatform/v1/index_endpoint.proto +++ b/google/cloud/aiplatform/v1/index_endpoint.proto @@ -233,7 +233,10 @@ message DeployedIndex { // // The value should be the name of the address // (https://cloud.google.com/compute/docs/reference/rest/v1/addresses) - // Example: 'vertex-ai-ip-range'. + // Example: ['vertex-ai-ip-range']. + // + // For more information about subnets and network IP ranges, please see + // https://cloud.google.com/vpc/docs/subnets#manually_created_subnet_ip_ranges. repeated string reserved_ip_ranges = 10 [(google.api.field_behavior) = OPTIONAL]; diff --git a/google/cloud/aiplatform/v1/machine_resources.proto b/google/cloud/aiplatform/v1/machine_resources.proto index ae19c491235ec..2a307e51f8090 100644 --- a/google/cloud/aiplatform/v1/machine_resources.proto +++ b/google/cloud/aiplatform/v1/machine_resources.proto @@ -181,6 +181,20 @@ message DiskSpec { int32 boot_disk_size_gb = 2; } +// Represents the spec of [persistent +// disk][https://cloud.google.com/compute/docs/disks/persistent-disks] options. +message PersistentDiskSpec { + // Type of the disk (default is "pd-standard"). + // Valid values: "pd-ssd" (Persistent Disk Solid State Drive) + // "pd-standard" (Persistent Disk Hard Disk Drive) + // "pd-balanced" (Balanced Persistent Disk) + // "pd-extreme" (Extreme Persistent Disk) + string disk_type = 1; + + // Size in GB of the disk (default is 100GB). + int64 disk_size_gb = 2; +} + // Represents a mount configuration for Network File System (NFS) to mount. message NfsMount { // Required. IP address of the NFS server. diff --git a/google/cloud/aiplatform/v1/model_monitoring.proto b/google/cloud/aiplatform/v1/model_monitoring.proto index 1978accfb378c..c747508d10d27 100644 --- a/google/cloud/aiplatform/v1/model_monitoring.proto +++ b/google/cloud/aiplatform/v1/model_monitoring.proto @@ -188,6 +188,13 @@ message ModelMonitoringAlertConfig { // This can be further sinked to Pub/Sub or any other services supported // by Cloud Logging. bool enable_logging = 2; + + // Resource names of the NotificationChannels to send alert. + // Must be of the format + // `projects//notificationChannels/` + repeated string notification_channels = 3 [(google.api.resource_reference) = { + type: "monitoring.googleapis.com/NotificationChannel" + }]; } // The config for feature monitoring threshold. diff --git a/google/cloud/aiplatform/v1/pipeline_job.proto b/google/cloud/aiplatform/v1/pipeline_job.proto index f5da296dae46b..e48cd95465381 100644 --- a/google/cloud/aiplatform/v1/pipeline_job.proto +++ b/google/cloud/aiplatform/v1/pipeline_job.proto @@ -199,7 +199,9 @@ message PipelineJob { // A template uri from where the // [PipelineJob.pipeline_spec][google.cloud.aiplatform.v1.PipelineJob.pipeline_spec], - // if empty, will be downloaded. + // if empty, will be downloaded. Currently, only uri from Vertex Template + // Registry & Gallery is supported. Reference to + // https://cloud.google.com/vertex-ai/docs/pipelines/create-pipeline-template. string template_uri = 19; // Output only. Pipeline template metadata. Will fill up fields if diff --git a/google/cloud/aiplatform/v1/pipeline_service.proto b/google/cloud/aiplatform/v1/pipeline_service.proto index 39acd31657c50..c500559d93181 100644 --- a/google/cloud/aiplatform/v1/pipeline_service.proto +++ b/google/cloud/aiplatform/v1/pipeline_service.proto @@ -312,7 +312,7 @@ message CreatePipelineJobRequest { // generated. // // This value should be less than 128 characters, and valid characters - // are /[a-z][0-9]-/. + // are `/[a-z][0-9]-/`. string pipeline_job_id = 3; } diff --git a/google/cloud/aiplatform/v1/tensorboard_service.proto b/google/cloud/aiplatform/v1/tensorboard_service.proto index 0086f73671dda..b5c769c0c1349 100644 --- a/google/cloud/aiplatform/v1/tensorboard_service.proto +++ b/google/cloud/aiplatform/v1/tensorboard_service.proto @@ -560,7 +560,7 @@ message CreateTensorboardExperimentRequest { // final component of the Tensorboard experiment's resource name. // // This value should be 1-128 characters, and valid characters - // are /[a-z][0-9]-/. + // are `/[a-z][0-9]-/`. string tensorboard_experiment_id = 3 [(google.api.field_behavior) = REQUIRED]; } @@ -710,7 +710,7 @@ message CreateTensorboardRunRequest { // component of the Tensorboard run's resource name. // // This value should be 1-128 characters, and valid characters - // are /[a-z][0-9]-/. + // are `/[a-z][0-9]-/`. string tensorboard_run_id = 3 [(google.api.field_behavior) = REQUIRED]; }