-
Notifications
You must be signed in to change notification settings - Fork 2.2k
/
import_config.proto
386 lines (334 loc) · 14.8 KB
/
import_config.proto
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
package google.cloud.retail.v2;
import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/retail/v2/product.proto";
import "google/cloud/retail/v2/user_event.proto";
import "google/protobuf/field_mask.proto";
import "google/protobuf/timestamp.proto";
import "google/rpc/status.proto";
import "google/type/date.proto";
option csharp_namespace = "Google.Cloud.Retail.V2";
option go_package = "cloud.google.com/go/retail/apiv2/retailpb;retailpb";
option java_multiple_files = true;
option java_outer_classname = "ImportConfigProto";
option java_package = "com.google.cloud.retail.v2";
option objc_class_prefix = "RETAIL";
option php_namespace = "Google\\Cloud\\Retail\\V2";
option ruby_package = "Google::Cloud::Retail::V2";
// Google Cloud Storage location for input content.
message GcsSource {
// Required. Google Cloud Storage URIs to input files. URI can be up to
// 2000 characters long. URIs can match the full object path (for example,
// `gs://bucket/directory/object.json`) or a pattern matching one or more
// files, such as `gs://bucket/directory/*.json`. A request can
// contain at most 100 files, and each file can be up to 2 GB. See
// [Importing product
// information](https://cloud.google.com/retail/recommendations-ai/docs/upload-catalog)
// for the expected file format and setup instructions.
repeated string input_uris = 1 [(google.api.field_behavior) = REQUIRED];
// The schema to use when parsing the data from the source.
//
// Supported values for product imports:
//
// * `product` (default): One JSON [Product][google.cloud.retail.v2.Product]
// per line. Each product must
// have a valid [Product.id][google.cloud.retail.v2.Product.id].
// * `product_merchant_center`: See [Importing catalog data from Merchant
// Center](https://cloud.google.com/retail/recommendations-ai/docs/upload-catalog#mc).
//
// Supported values for user events imports:
//
// * `user_event` (default): One JSON
// [UserEvent][google.cloud.retail.v2.UserEvent] per line.
// * `user_event_ga360`: Using
// https://support.google.com/analytics/answer/3437719.
//
// Supported values for control imports:
//
// * `control` (default): One JSON [Control][google.cloud.retail.v2.Control]
// per line.
//
// Supported values for catalog attribute imports:
//
// * `catalog_attribute` (default): One CSV
// [CatalogAttribute][google.cloud.retail.v2.CatalogAttribute] per line.
string data_schema = 2;
}
// BigQuery source import data from.
message BigQuerySource {
// BigQuery table partition info. Leave this empty if the BigQuery table
// is not partitioned.
oneof partition {
// BigQuery time partitioned table's _PARTITIONDATE in YYYY-MM-DD format.
//
// Only supported in
// [ImportProductsRequest][google.cloud.retail.v2.ImportProductsRequest].
google.type.Date partition_date = 6;
}
// The project ID (can be project # or ID) that the BigQuery source is in with
// a length limit of 128 characters. If not specified, inherits the project
// ID from the parent request.
string project_id = 5;
// Required. The BigQuery data set to copy the data from with a length limit
// of 1,024 characters.
string dataset_id = 1 [(google.api.field_behavior) = REQUIRED];
// Required. The BigQuery table to copy the data from with a length limit of
// 1,024 characters.
string table_id = 2 [(google.api.field_behavior) = REQUIRED];
// Intermediate Cloud Storage directory used for the import with a length
// limit of 2,000 characters. Can be specified if one wants to have the
// BigQuery export to a specific Cloud Storage directory.
string gcs_staging_dir = 3;
// The schema to use when parsing the data from the source.
//
// Supported values for product imports:
//
// * `product` (default): One JSON [Product][google.cloud.retail.v2.Product]
// per line. Each product must
// have a valid [Product.id][google.cloud.retail.v2.Product.id].
// * `product_merchant_center`: See [Importing catalog data from Merchant
// Center](https://cloud.google.com/retail/recommendations-ai/docs/upload-catalog#mc).
//
// Supported values for user events imports:
//
// * `user_event` (default): One JSON
// [UserEvent][google.cloud.retail.v2.UserEvent] per line.
// * `user_event_ga360`:
// The schema is available here:
// https://support.google.com/analytics/answer/3437719.
// * `user_event_ga4`:
// The schema is available here:
// https://support.google.com/analytics/answer/7029846.
//
// Supported values for autocomplete imports:
//
// * `suggestions` (default): One JSON completion suggestion per line.
// * `denylist`: One JSON deny suggestion per line.
// * `allowlist`: One JSON allow suggestion per line.
string data_schema = 4;
}
// The inline source for the input config for ImportProducts method.
message ProductInlineSource {
// Required. A list of products to update/create. Each product must have a
// valid [Product.id][google.cloud.retail.v2.Product.id]. Recommended max of
// 100 items.
repeated Product products = 1 [(google.api.field_behavior) = REQUIRED];
}
// The inline source for the input config for ImportUserEvents method.
message UserEventInlineSource {
// Required. A list of user events to import. Recommended max of 10k items.
repeated UserEvent user_events = 1 [(google.api.field_behavior) = REQUIRED];
}
// Configuration of destination for Import related errors.
message ImportErrorsConfig {
// Required. Errors destination.
oneof destination {
// Google Cloud Storage prefix for import errors. This must be an empty,
// existing Cloud Storage directory. Import errors are written to
// sharded files in this directory, one per line, as a JSON-encoded
// `google.rpc.Status` message.
string gcs_prefix = 1;
}
}
// Request message for Import methods.
message ImportProductsRequest {
// Indicates how imported products are reconciled with the existing products
// created or imported before.
enum ReconciliationMode {
// Defaults to INCREMENTAL.
RECONCILIATION_MODE_UNSPECIFIED = 0;
// Inserts new products or updates existing products.
INCREMENTAL = 1;
// Calculates diff and replaces the entire product dataset. Existing
// products may be deleted if they are not present in the source location.
FULL = 2;
}
// Required.
// `projects/1234/locations/global/catalogs/default_catalog/branches/default_branch`
//
// If no updateMask is specified, requires products.create permission.
// If updateMask is specified, requires products.update permission.
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = { type: "retail.googleapis.com/Branch" }
];
// Deprecated. This field has no effect.
string request_id = 6 [deprecated = true];
// Required. The desired input location of the data.
ProductInputConfig input_config = 2 [(google.api.field_behavior) = REQUIRED];
// The desired location of errors incurred during the Import.
ImportErrorsConfig errors_config = 3;
// Indicates which fields in the provided imported `products` to update. If
// not set, all fields are updated.
google.protobuf.FieldMask update_mask = 4;
// The mode of reconciliation between existing products and the products to be
// imported. Defaults to
// [ReconciliationMode.INCREMENTAL][google.cloud.retail.v2.ImportProductsRequest.ReconciliationMode.INCREMENTAL].
ReconciliationMode reconciliation_mode = 5;
// Full Pub/Sub topic name for receiving notification. If this field is set,
// when the import is finished, a notification is sent to
// specified Pub/Sub topic. The message data is JSON string of a
// [Operation][google.longrunning.Operation].
//
// Format of the Pub/Sub topic is `projects/{project}/topics/{topic}`. It has
// to be within the same project as
// [ImportProductsRequest.parent][google.cloud.retail.v2.ImportProductsRequest.parent].
// Make sure that `service-<project
// number>@gcp-sa-retail.iam.gserviceaccount.com` has the
// `pubsub.topics.publish` IAM permission on the topic.
string notification_pubsub_topic = 7;
}
// Request message for the ImportUserEvents request.
message ImportUserEventsRequest {
// Required. `projects/1234/locations/global/catalogs/default_catalog`
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = { type: "retail.googleapis.com/Catalog" }
];
// Required. The desired input location of the data.
UserEventInputConfig input_config = 2
[(google.api.field_behavior) = REQUIRED];
// The desired location of errors incurred during the Import. Cannot be set
// for inline user event imports.
ImportErrorsConfig errors_config = 3;
}
// Request message for ImportCompletionData methods.
message ImportCompletionDataRequest {
// Required. The catalog which the suggestions dataset belongs to.
//
// Format: `projects/1234/locations/global/catalogs/default_catalog`.
string parent = 1 [
(google.api.field_behavior) = REQUIRED,
(google.api.resource_reference) = { type: "retail.googleapis.com/Catalog" }
];
// Required. The desired input location of the data.
CompletionDataInputConfig input_config = 2
[(google.api.field_behavior) = REQUIRED];
// Pub/Sub topic for receiving notification. If this field is set,
// when the import is finished, a notification is sent to
// specified Pub/Sub topic. The message data is JSON string of a
// [Operation][google.longrunning.Operation].
// Format of the Pub/Sub topic is `projects/{project}/topics/{topic}`.
string notification_pubsub_topic = 3;
}
// The input config source for products.
message ProductInputConfig {
// Required. The source of the input.
oneof source {
// The Inline source for the input content for products.
ProductInlineSource product_inline_source = 1;
// Google Cloud Storage location for the input content.
GcsSource gcs_source = 2;
// BigQuery input source.
BigQuerySource big_query_source = 3;
}
}
// The input config source for user events.
message UserEventInputConfig {
// The source of the input.
oneof source {
// Required. The Inline source for the input content for UserEvents.
UserEventInlineSource user_event_inline_source = 1
[(google.api.field_behavior) = REQUIRED];
// Required. Google Cloud Storage location for the input content.
GcsSource gcs_source = 2 [(google.api.field_behavior) = REQUIRED];
// Required. BigQuery input source.
BigQuerySource big_query_source = 3
[(google.api.field_behavior) = REQUIRED];
}
}
// The input config source for completion data.
message CompletionDataInputConfig {
// The source of the input.
//
// Supported
// [BigQuerySource.data_schema][google.cloud.retail.v2.BigQuerySource.data_schema]
// values for suggestions imports:
//
// * `suggestions` (default): One JSON completion suggestion per line.
// * `denylist`: One JSON deny suggestion per line.
// * `allowlist`: One JSON allow suggestion per line.
oneof source {
// Required. BigQuery input source.
//
// Add the IAM permission "BigQuery Data Viewer" for
// cloud-retail-customer-data-access@system.gserviceaccount.com before
// using this feature otherwise an error is thrown.
BigQuerySource big_query_source = 1
[(google.api.field_behavior) = REQUIRED];
}
}
// Metadata related to the progress of the Import operation. This is
// returned by the google.longrunning.Operation.metadata field.
message ImportMetadata {
// Operation create time.
google.protobuf.Timestamp create_time = 1;
// Operation last update time. If the operation is done, this is also the
// finish time.
google.protobuf.Timestamp update_time = 2;
// Count of entries that were processed successfully.
int64 success_count = 3;
// Count of entries that encountered errors while processing.
int64 failure_count = 4;
// Deprecated. This field is never set.
string request_id = 5 [deprecated = true];
// Pub/Sub topic for receiving notification. If this field is set,
// when the import is finished, a notification is sent to
// specified Pub/Sub topic. The message data is JSON string of a
// [Operation][google.longrunning.Operation].
// Format of the Pub/Sub topic is `projects/{project}/topics/{topic}`.
string notification_pubsub_topic = 6;
}
// Response of the
// [ImportProductsRequest][google.cloud.retail.v2.ImportProductsRequest]. If the
// long running operation is done, then this message is returned by the
// google.longrunning.Operations.response field if the operation was successful.
message ImportProductsResponse {
// A sample of errors encountered while processing the request.
repeated google.rpc.Status error_samples = 1;
// Echoes the destination for the complete errors in the request if set.
ImportErrorsConfig errors_config = 2;
}
// Response of the ImportUserEventsRequest. If the long running
// operation was successful, then this message is returned by the
// google.longrunning.Operations.response field if the operation was successful.
message ImportUserEventsResponse {
// A sample of errors encountered while processing the request.
repeated google.rpc.Status error_samples = 1;
// Echoes the destination for the complete errors if this field was set in
// the request.
ImportErrorsConfig errors_config = 2;
// Aggregated statistics of user event import status.
UserEventImportSummary import_summary = 3;
}
// A summary of import result. The UserEventImportSummary summarizes
// the import status for user events.
message UserEventImportSummary {
// Count of user events imported with complete existing catalog information.
int64 joined_events_count = 1;
// Count of user events imported, but with catalog information not found
// in the imported catalog.
int64 unjoined_events_count = 2;
}
// Response of the
// [ImportCompletionDataRequest][google.cloud.retail.v2.ImportCompletionDataRequest].
// If the long running operation is done, this message is returned by the
// google.longrunning.Operations.response field if the operation is successful.
message ImportCompletionDataResponse {
// A sample of errors encountered while processing the request.
repeated google.rpc.Status error_samples = 1;
}