Skip to content

Commit

Permalink
feat: support blended search
Browse files Browse the repository at this point in the history
feat: support boost in multi-turn search
feat: add document processing config services
feat: add search tuning services
feat: allow setting schema on schema creation
docs: keep the API doc up-to-date with recent changes

PiperOrigin-RevId: 614046598
  • Loading branch information
Google APIs authored and Copybara-Service committed Mar 8, 2024
1 parent 252bef0 commit d66613f
Show file tree
Hide file tree
Showing 11 changed files with 341 additions and 13 deletions.
22 changes: 21 additions & 1 deletion google/cloud/discoveryengine/v1beta/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,17 @@
# * extra_protoc_file_parameters
# The complete list of preserved parameters can be found in the source code.

# buildifier: disable=load-on-top

# This is an API workspace, having public visibility by default makes perfect sense.
package(default_visibility = ["//visibility:public"])

##############################################################################
# Common
##############################################################################
load("@rules_proto//proto:defs.bzl", "proto_library")
# buildifier: disable=same-origin-load
load("@com_google_googleapis_imports//:imports.bzl", "proto_library_with_info")
load("@rules_proto//proto:defs.bzl", "proto_library")

proto_library(
name = "discoveryengine_proto",
Expand All @@ -29,6 +32,7 @@ proto_library(
"data_store.proto",
"data_store_service.proto",
"document.proto",
"document_processing_config.proto",
"document_service.proto",
"engine.proto",
"engine_service.proto",
Expand All @@ -38,6 +42,7 @@ proto_library(
"schema.proto",
"schema_service.proto",
"search_service.proto",
"search_tuning_service.proto",
"serving_config.proto",
"serving_config_service.proto",
"site_search_engine.proto",
Expand Down Expand Up @@ -67,12 +72,14 @@ proto_library_with_info(
deps = [
":discoveryengine_proto",
"//google/cloud:common_resources_proto",
"//google/cloud/location:location_proto",
],
)

##############################################################################
# Java
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"java_gapic_assembly_gradle_pkg",
Expand Down Expand Up @@ -102,11 +109,13 @@ java_gapic_library(
service_yaml = "discoveryengine_v1beta.yaml",
test_deps = [
":discoveryengine_java_grpc",
"//google/cloud/location:location_java_grpc",
],
transport = "grpc+rest",
deps = [
":discoveryengine_java_proto",
"//google/api:api_java_proto",
"//google/cloud/location:location_java_proto",
],
)

Expand All @@ -129,6 +138,8 @@ java_gapic_test(
"com.google.cloud.discoveryengine.v1beta.SchemaServiceClientTest",
"com.google.cloud.discoveryengine.v1beta.SearchServiceClientHttpJsonTest",
"com.google.cloud.discoveryengine.v1beta.SearchServiceClientTest",
"com.google.cloud.discoveryengine.v1beta.SearchTuningServiceClientHttpJsonTest",
"com.google.cloud.discoveryengine.v1beta.SearchTuningServiceClientTest",
"com.google.cloud.discoveryengine.v1beta.ServingConfigServiceClientHttpJsonTest",
"com.google.cloud.discoveryengine.v1beta.ServingConfigServiceClientTest",
"com.google.cloud.discoveryengine.v1beta.SiteSearchEngineServiceClientHttpJsonTest",
Expand All @@ -155,6 +166,7 @@ java_gapic_assembly_gradle_pkg(
##############################################################################
# Go
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"go_gapic_assembly_pkg",
Expand Down Expand Up @@ -189,6 +201,7 @@ go_gapic_library(
deps = [
":discoveryengine_go_proto",
"//google/api:httpbody_go_proto",
"//google/cloud/location:location_go_proto",
"//google/longrunning:longrunning_go_proto",
"@com_google_cloud_go_longrunning//:go_default_library",
"@com_google_cloud_go_longrunning//autogen:go_default_library",
Expand All @@ -212,6 +225,7 @@ go_gapic_assembly_pkg(
##############################################################################
# Python
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"py_gapic_assembly_pkg",
Expand Down Expand Up @@ -251,6 +265,7 @@ py_gapic_assembly_pkg(
##############################################################################
# PHP
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"php_gapic_assembly_pkg",
Expand Down Expand Up @@ -288,6 +303,7 @@ php_gapic_assembly_pkg(
##############################################################################
# Node.js
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"nodejs_gapic_assembly_pkg",
Expand Down Expand Up @@ -318,6 +334,7 @@ nodejs_gapic_assembly_pkg(
##############################################################################
# Ruby
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"ruby_cloud_gapic_library",
Expand Down Expand Up @@ -364,6 +381,7 @@ ruby_gapic_assembly_pkg(
##############################################################################
# C#
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"csharp_gapic_assembly_pkg",
Expand All @@ -374,6 +392,7 @@ load(

csharp_proto_library(
name = "discoveryengine_csharp_proto",
extra_opts = [],
deps = [":discoveryengine_proto"],
)

Expand Down Expand Up @@ -410,6 +429,7 @@ csharp_gapic_assembly_pkg(
##############################################################################
# C++
##############################################################################
# buildifier: disable=same-origin-load
load(
"@com_google_googleapis_imports//:imports.bzl",
"cc_grpc_library",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,11 @@ message ConverseConversationRequest {
// operators, see
// [Filter](https://cloud.google.com/generative-ai-app-builder/docs/filter-search-metadata)
string filter = 9;

// Boost specification to boost certain documents in search results which may
// affect the converse response. For more information on boosting, see
// [Boosting](https://cloud.google.com/retail/docs/boosting#boost)
SearchRequest.BoostSpec boost_spec = 10;
}

// Response message for
Expand Down
22 changes: 22 additions & 0 deletions google/cloud/discoveryengine/v1beta/data_store.proto
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package google.cloud.discoveryengine.v1beta;
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/discoveryengine/v1beta/common.proto";
import "google/cloud/discoveryengine/v1beta/document_processing_config.proto";
import "google/cloud/discoveryengine/v1beta/schema.proto";
import "google/protobuf/timestamp.proto";

option csharp_namespace = "Google.Cloud.DiscoveryEngine.V1Beta";
Expand Down Expand Up @@ -95,4 +97,24 @@ message DataStore {
// [DataStore][google.cloud.discoveryengine.v1beta.DataStore] was created at.
google.protobuf.Timestamp create_time = 4
[(google.api.field_behavior) = OUTPUT_ONLY];

// Configuration for Document understanding and enrichment.
DocumentProcessingConfig document_processing_config = 27;

// The start schema to use for this
// [DataStore][google.cloud.discoveryengine.v1beta.DataStore] when
// provisioning it. If unset, a default vertical specialized schema will be
// used.
//
// This field is only used by [CreateDataStore][] API, and will be ignored if
// used in other APIs. This field will be omitted from all API responses
// including [CreateDataStore][] API. To retrieve a schema of a
// [DataStore][google.cloud.discoveryengine.v1beta.DataStore], use
// [SchemaService.GetSchema][google.cloud.discoveryengine.v1beta.SchemaService.GetSchema]
// API instead.
//
// The provided schema will be validated against certain rules on schema.
// Learn more from [this
// doc](https://cloud.google.com/generative-ai-app-builder/docs/provide-schema).
Schema starting_schema = 28;
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ apis:
- name: google.cloud.discoveryengine.v1beta.RecommendationService
- name: google.cloud.discoveryengine.v1beta.SchemaService
- name: google.cloud.discoveryengine.v1beta.SearchService
- name: google.cloud.discoveryengine.v1beta.SearchTuningService
- name: google.cloud.discoveryengine.v1beta.ServingConfigService
- name: google.cloud.discoveryengine.v1beta.SiteSearchEngineService
- name: google.cloud.discoveryengine.v1beta.UserEventService
Expand Down Expand Up @@ -48,6 +49,8 @@ types:
- name: google.cloud.discoveryengine.v1beta.PurgeSuggestionDenyListEntriesResponse
- name: google.cloud.discoveryengine.v1beta.Schema
- name: google.cloud.discoveryengine.v1beta.TargetSite
- name: google.cloud.discoveryengine.v1beta.TrainCustomModelMetadata
- name: google.cloud.discoveryengine.v1beta.TrainCustomModelResponse
- name: google.cloud.discoveryengine.v1beta.UpdateSchemaMetadata
- name: google.cloud.discoveryengine.v1beta.UpdateTargetSiteMetadata

Expand Down Expand Up @@ -123,6 +126,10 @@ authentication:
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/cloud-platform
- selector: google.cloud.discoveryengine.v1beta.SearchTuningService.TrainCustomModel
oauth:
canonical_scopes: |-
https://www.googleapis.com/auth/cloud-platform
- selector: 'google.cloud.discoveryengine.v1beta.ServingConfigService.*'
oauth:
canonical_scopes: |-
Expand Down
9 changes: 9 additions & 0 deletions google/cloud/discoveryengine/v1beta/document.proto
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package google.cloud.discoveryengine.v1beta;
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";

option csharp_namespace = "Google.Cloud.DiscoveryEngine.V1Beta";
option go_package = "cloud.google.com/go/discoveryengine/apiv1beta/discoveryenginepb;discoveryenginepb";
Expand Down Expand Up @@ -118,4 +119,12 @@ message Document {
// It contains derived data that are not in the original input document.
google.protobuf.Struct derived_struct_data = 6
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The last time the document was indexed. If this field is set,
// the document could be returned in search results.
//
// This field is OUTPUT_ONLY. If this field is not populated, it means the
// document has never been indexed.
google.protobuf.Timestamp index_time = 13
[(google.api.field_behavior) = OUTPUT_ONLY];
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.discoveryengine.v1beta;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";

option csharp_namespace = "Google.Cloud.DiscoveryEngine.V1Beta";
option go_package = "cloud.google.com/go/discoveryengine/apiv1beta/discoveryenginepb;discoveryenginepb";
option java_multiple_files = true;
option java_outer_classname = "DocumentProcessingConfigProto";
option java_package = "com.google.cloud.discoveryengine.v1beta";
option objc_class_prefix = "DISCOVERYENGINE";
option php_namespace = "Google\\Cloud\\DiscoveryEngine\\V1beta";
option ruby_package = "Google::Cloud::DiscoveryEngine::V1beta";

// A singleton resource of
// [DataStore][google.cloud.discoveryengine.v1beta.DataStore]. It's empty when
// [DataStore][google.cloud.discoveryengine.v1beta.DataStore] is created, which
// defaults to digital parser. The first call to
// [DataStoreService.UpdateDocumentProcessingConfig][] method will initialize
// the config.
message DocumentProcessingConfig {
option (google.api.resource) = {
type: "discoveryengine.googleapis.com/DocumentProcessingConfig"
pattern: "projects/{project}/locations/{location}/dataStores/{data_store}/documentProcessingConfig"
pattern: "projects/{project}/locations/{location}/collections/{collection}/dataStores/{data_store}/documentProcessingConfig"
};

// Related configurations applied to a specific type of document parser.
message ParsingConfig {
// The digital parsing configurations for documents.
message DigitalParsingConfig {}

// The OCR parsing configurations for documents.
message OcrParsingConfig {
// Apply additional enhanced OCR processing to a list of document
// elements.
//
// Supported values:
// * `table`: advanced table parsing model.
repeated string enhanced_document_elements = 1;

// If true, will use native text instead of OCR text on pages containing
// native text.
bool use_native_text = 2;
}

// Configs for document processing types.
oneof type_dedicated_config {
// Configurations applied to digital parser.
DigitalParsingConfig digital_parsing_config = 1;

// Configurations applied to OCR parser. Currently it only applies to
// PDFs.
OcrParsingConfig ocr_parsing_config = 2;
}
}

// The full resource name of the Document Processing Config.
// Format:
// `projects/*/locations/*/collections/*/dataStores/*/documentProcessingConfig`.
string name = 1;

// Configurations for default Document parser.
// If not specified, we will configure it as default DigitalParsingConfig, and
// the default parsing config will be applied to all file types for Document
// parsing.
ParsingConfig default_parsing_config = 4;

// Map from file type to override the default parsing configuration based on
// the file type. Supported keys:
// * `pdf`: Override parsing config for PDF files, either digital parsing, ocr
// parsing or layout parsing is supported.
// * `html`: Override parsing config for HTML files, only digital parsing and
// or layout parsing are supported.
// * `docx`: Override parsing config for DOCX files, only digital parsing and
// or layout parsing are supported.
map<string, ParsingConfig> parsing_config_overrides = 5;
}
3 changes: 3 additions & 0 deletions google/cloud/discoveryengine/v1beta/purge_config.proto
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ message PurgeDocumentsMetadata {

// Count of entries that encountered errors while processing.
int64 failure_count = 4;

// Count of entries that were ignored as entries were not found.
int64 ignored_count = 5;
}

// Request message for
Expand Down
23 changes: 19 additions & 4 deletions google/cloud/discoveryengine/v1beta/search_service.proto
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,19 @@ message SearchRequest {
}
}

// A struct to define data stores to filter on in a search call.
message DataStoreSpec {
// Required. Full resource name of
// [DataStore][google.cloud.discoveryengine.v1beta.DataStore], such as
// `projects/{project}/locations/{location}/collections/{collection_id}/dataStores/{data_store_id}`.
string data_store = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = {
type: "discoveryengine.googleapis.com/DataStore"
}
];
}

// A facet specification to perform faceted search.
message FacetSpec {
// Specifies how a facet is computed.
Expand Down Expand Up @@ -212,9 +225,8 @@ message SearchRequest {
// Examples:
//
// * To boost documents with document ID "doc_1" or "doc_2", and
// color
// "Red" or "Blue":
// * (id: ANY("doc_1", "doc_2")) AND (color: ANY("Red","Blue"))
// color "Red" or "Blue":
// `(document_id: ANY("doc_1", "doc_2")) AND (color: ANY("Red", "Blue"))`
string condition = 1;

// Strength of the condition boost, which should be in [-1, 1]. Negative
Expand Down Expand Up @@ -347,7 +359,7 @@ message SearchRequest {
// of results returned is less than `summaryResultCount`, the summary is
// generated from all of the results.
//
// At most five results can be used to generate a summary.
// At most 10 results can be used to generate a summary.
int32 summary_result_count = 1;

// Specifies whether to include citations in the summary. The default
Expand Down Expand Up @@ -547,6 +559,9 @@ message SearchRequest {
// If this field is negative, an `INVALID_ARGUMENT` is returned.
int32 offset = 6;

// A list of data store specs to apply on a search call.
repeated DataStoreSpec data_store_specs = 32;

// The filter syntax consists of an expression language for constructing a
// predicate from one or more fields of the documents being filtered. Filter
// expression is case-sensitive.
Expand Down

0 comments on commit d66613f

Please sign in to comment.