Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
731 lines (613 sloc) 27 KB
// Copyright 2019 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
syntax = "proto3";
package google.cloud.bigquery.datatransfer.v1;
import "google/api/annotations.proto";
import "google/cloud/bigquery/datatransfer/v1/transfer.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/empty.proto";
import "google/protobuf/field_mask.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/wrappers.proto";
import "google/api/client.proto";
option csharp_namespace = "Google.Cloud.BigQuery.DataTransfer.V1";
option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/datatransfer/v1;datatransfer";
option java_multiple_files = true;
option java_outer_classname = "DataTransferProto";
option java_package = "com.google.cloud.bigquery.datatransfer.v1";
option php_namespace = "Google\\Cloud\\BigQuery\\DataTransfer\\V1";
// The Google BigQuery Data Transfer Service API enables BigQuery users to
// configure the transfer of their data from other Google Products into
// BigQuery. This service contains methods that are end user exposed. It backs
// up the frontend.
service DataTransferService {
option (google.api.default_host) = "bigquerydatatransfer.googleapis.com";
option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform";
// Retrieves a supported data source and returns its settings,
// which can be used for UI rendering.
rpc GetDataSource(GetDataSourceRequest) returns (DataSource) {
option (google.api.http) = {
get: "/v1/{name=projects/*/locations/*/dataSources/*}"
additional_bindings {
get: "/v1/{name=projects/*/dataSources/*}"
}
};
}
// Lists supported data sources and returns their settings,
// which can be used for UI rendering.
rpc ListDataSources(ListDataSourcesRequest) returns (ListDataSourcesResponse) {
option (google.api.http) = {
get: "/v1/{parent=projects/*/locations/*}/dataSources"
additional_bindings {
get: "/v1/{parent=projects/*}/dataSources"
}
};
}
// Creates a new data transfer configuration.
rpc CreateTransferConfig(CreateTransferConfigRequest) returns (TransferConfig) {
option (google.api.http) = {
post: "/v1/{parent=projects/*/locations/*}/transferConfigs"
body: "transfer_config"
additional_bindings {
post: "/v1/{parent=projects/*}/transferConfigs"
body: "transfer_config"
}
};
}
// Updates a data transfer configuration.
// All fields must be set, even if they are not updated.
rpc UpdateTransferConfig(UpdateTransferConfigRequest) returns (TransferConfig) {
option (google.api.http) = {
patch: "/v1/{transfer_config.name=projects/*/locations/*/transferConfigs/*}"
body: "transfer_config"
additional_bindings {
patch: "/v1/{transfer_config.name=projects/*/transferConfigs/*}"
body: "transfer_config"
}
};
}
// Deletes a data transfer configuration,
// including any associated transfer runs and logs.
rpc DeleteTransferConfig(DeleteTransferConfigRequest) returns (google.protobuf.Empty) {
option (google.api.http) = {
delete: "/v1/{name=projects/*/locations/*/transferConfigs/*}"
additional_bindings {
delete: "/v1/{name=projects/*/transferConfigs/*}"
}
};
}
// Returns information about a data transfer config.
rpc GetTransferConfig(GetTransferConfigRequest) returns (TransferConfig) {
option (google.api.http) = {
get: "/v1/{name=projects/*/locations/*/transferConfigs/*}"
additional_bindings {
get: "/v1/{name=projects/*/transferConfigs/*}"
}
};
}
// Returns information about all data transfers in the project.
rpc ListTransferConfigs(ListTransferConfigsRequest) returns (ListTransferConfigsResponse) {
option (google.api.http) = {
get: "/v1/{parent=projects/*/locations/*}/transferConfigs"
additional_bindings {
get: "/v1/{parent=projects/*}/transferConfigs"
}
};
}
// Creates transfer runs for a time range [start_time, end_time].
// For each date - or whatever granularity the data source supports - in the
// range, one transfer run is created.
// Note that runs are created per UTC time in the time range.
// DEPRECATED: use StartManualTransferRuns instead.
rpc ScheduleTransferRuns(ScheduleTransferRunsRequest) returns (ScheduleTransferRunsResponse) {
option deprecated = true;
option (google.api.http) = {
post: "/v1/{parent=projects/*/locations/*/transferConfigs/*}:scheduleRuns"
body: "*"
additional_bindings {
post: "/v1/{parent=projects/*/transferConfigs/*}:scheduleRuns"
body: "*"
}
};
}
// Start manual transfer runs to be executed now with schedule_time equal to
// current time. The transfer runs can be created for a time range where the
// run_time is between start_time (inclusive) and end_time (exclusive), or for
// a specific run_time.
rpc StartManualTransferRuns(StartManualTransferRunsRequest) returns (StartManualTransferRunsResponse) {
option (google.api.http) = {
post: "/v1/{parent=projects/*/locations/*/transferConfigs/*}:startManualRuns"
body: "*"
additional_bindings {
post: "/v1/{parent=projects/*/transferConfigs/*}:startManualRuns"
body: "*"
}
};
}
// Returns information about the particular transfer run.
rpc GetTransferRun(GetTransferRunRequest) returns (TransferRun) {
option (google.api.http) = {
get: "/v1/{name=projects/*/locations/*/transferConfigs/*/runs/*}"
additional_bindings {
get: "/v1/{name=projects/*/transferConfigs/*/runs/*}"
}
};
}
// Deletes the specified transfer run.
rpc DeleteTransferRun(DeleteTransferRunRequest) returns (google.protobuf.Empty) {
option (google.api.http) = {
delete: "/v1/{name=projects/*/locations/*/transferConfigs/*/runs/*}"
additional_bindings {
delete: "/v1/{name=projects/*/transferConfigs/*/runs/*}"
}
};
}
// Returns information about running and completed jobs.
rpc ListTransferRuns(ListTransferRunsRequest) returns (ListTransferRunsResponse) {
option (google.api.http) = {
get: "/v1/{parent=projects/*/locations/*/transferConfigs/*}/runs"
additional_bindings {
get: "/v1/{parent=projects/*/transferConfigs/*}/runs"
}
};
}
// Returns user facing log messages for the data transfer run.
rpc ListTransferLogs(ListTransferLogsRequest) returns (ListTransferLogsResponse) {
option (google.api.http) = {
get: "/v1/{parent=projects/*/locations/*/transferConfigs/*/runs/*}/transferLogs"
additional_bindings {
get: "/v1/{parent=projects/*/transferConfigs/*/runs/*}/transferLogs"
}
};
}
// Returns true if valid credentials exist for the given data source and
// requesting user.
// Some data sources doesn't support service account, so we need to talk to
// them on behalf of the end user. This API just checks whether we have OAuth
// token for the particular user, which is a pre-requisite before user can
// create a transfer config.
rpc CheckValidCreds(CheckValidCredsRequest) returns (CheckValidCredsResponse) {
option (google.api.http) = {
post: "/v1/{name=projects/*/locations/*/dataSources/*}:checkValidCreds"
body: "*"
additional_bindings {
post: "/v1/{name=projects/*/dataSources/*}:checkValidCreds"
body: "*"
}
};
}
}
// Represents a data source parameter with validation rules, so that
// parameters can be rendered in the UI. These parameters are given to us by
// supported data sources, and include all needed information for rendering
// and validation.
// Thus, whoever uses this api can decide to generate either generic ui,
// or custom data source specific forms.
message DataSourceParameter {
// Parameter type.
enum Type {
// Type unspecified.
TYPE_UNSPECIFIED = 0;
// String parameter.
STRING = 1;
// Integer parameter (64-bits).
// Will be serialized to json as string.
INTEGER = 2;
// Double precision floating point parameter.
DOUBLE = 3;
// Boolean parameter.
BOOLEAN = 4;
// Deprecated. This field has no effect.
RECORD = 5;
// Page ID for a Google+ Page.
PLUS_PAGE = 6;
}
// Parameter identifier.
string param_id = 1;
// Parameter display name in the user interface.
string display_name = 2;
// Parameter description.
string description = 3;
// Parameter type.
Type type = 4;
// Is parameter required.
bool required = 5;
// Deprecated. This field has no effect.
bool repeated = 6;
// Regular expression which can be used for parameter validation.
string validation_regex = 7;
// All possible values for the parameter.
repeated string allowed_values = 8;
// For integer and double values specifies minimum allowed value.
google.protobuf.DoubleValue min_value = 9;
// For integer and double values specifies maxminum allowed value.
google.protobuf.DoubleValue max_value = 10;
// Deprecated. This field has no effect.
repeated DataSourceParameter fields = 11;
// Description of the requirements for this field, in case the user input does
// not fulfill the regex pattern or min/max values.
string validation_description = 12;
// URL to a help document to further explain the naming requirements.
string validation_help_url = 13;
// Cannot be changed after initial creation.
bool immutable = 14;
// Deprecated. This field has no effect.
bool recurse = 15;
// If true, it should not be used in new transfers, and it should not be
// visible to users.
bool deprecated = 20;
}
// Represents data source metadata. Metadata is sufficient to
// render UI and request proper OAuth tokens.
message DataSource {
// The type of authorization needed for this data source.
enum AuthorizationType {
// Type unspecified.
AUTHORIZATION_TYPE_UNSPECIFIED = 0;
// Use OAuth 2 authorization codes that can be exchanged
// for a refresh token on the backend.
AUTHORIZATION_CODE = 1;
// Return an authorization code for a given Google+ page that can then be
// exchanged for a refresh token on the backend.
GOOGLE_PLUS_AUTHORIZATION_CODE = 2;
}
// Represents how the data source supports data auto refresh.
enum DataRefreshType {
// The data source won't support data auto refresh, which is default value.
DATA_REFRESH_TYPE_UNSPECIFIED = 0;
// The data source supports data auto refresh, and runs will be scheduled
// for the past few days. Does not allow custom values to be set for each
// transfer config.
SLIDING_WINDOW = 1;
// The data source supports data auto refresh, and runs will be scheduled
// for the past few days. Allows custom values to be set for each transfer
// config.
CUSTOM_SLIDING_WINDOW = 2;
}
// Output only. Data source resource name.
string name = 1;
// Data source id.
string data_source_id = 2;
// User friendly data source name.
string display_name = 3;
// User friendly data source description string.
string description = 4;
// Data source client id which should be used to receive refresh token.
string client_id = 5;
// Api auth scopes for which refresh token needs to be obtained. These are
// scopes needed by a data source to prepare data and ingest them into
// BigQuery, e.g., https://www.googleapis.com/auth/bigquery
repeated string scopes = 6;
// Deprecated. This field has no effect.
TransferType transfer_type = 7 [deprecated = true];
// Deprecated. This field has no effect.
bool supports_multiple_transfers = 8 [deprecated = true];
// The number of seconds to wait for an update from the data source
// before the Data Transfer Service marks the transfer as FAILED.
int32 update_deadline_seconds = 9;
// Default data transfer schedule.
// Examples of valid schedules include:
// `1st,3rd monday of month 15:30`,
// `every wed,fri of jan,jun 13:15`, and
// `first sunday of quarter 00:00`.
string default_schedule = 10;
// Specifies whether the data source supports a user defined schedule, or
// operates on the default schedule.
// When set to `true`, user can override default schedule.
bool supports_custom_schedule = 11;
// Data source parameters.
repeated DataSourceParameter parameters = 12;
// Url for the help document for this data source.
string help_url = 13;
// Indicates the type of authorization.
AuthorizationType authorization_type = 14;
// Specifies whether the data source supports automatic data refresh for the
// past few days, and how it's supported.
// For some data sources, data might not be complete until a few days later,
// so it's useful to refresh data automatically.
DataRefreshType data_refresh_type = 15;
// Default data refresh window on days.
// Only meaningful when `data_refresh_type` = `SLIDING_WINDOW`.
int32 default_data_refresh_window_days = 16;
// Disables backfilling and manual run scheduling
// for the data source.
bool manual_runs_disabled = 17;
// The minimum interval for scheduler to schedule runs.
google.protobuf.Duration minimum_schedule_interval = 18;
}
// A request to get data source info.
message GetDataSourceRequest {
// The field will contain name of the resource requested, for example:
// `projects/{project_id}/dataSources/{data_source_id}`
string name = 1;
}
// Request to list supported data sources and their data transfer settings.
message ListDataSourcesRequest {
// The BigQuery project id for which data sources should be returned.
// Must be in the form: `projects/{project_id}`
string parent = 1;
// Pagination token, which can be used to request a specific page
// of `ListDataSourcesRequest` list results. For multiple-page
// results, `ListDataSourcesResponse` outputs
// a `next_page` token, which can be used as the
// `page_token` value to request the next page of list results.
string page_token = 3;
// Page size. The default page size is the maximum value of 1000 results.
int32 page_size = 4;
}
// Returns list of supported data sources and their metadata.
message ListDataSourcesResponse {
// List of supported data sources and their transfer settings.
repeated DataSource data_sources = 1;
// Output only. The next-pagination token. For multiple-page list results,
// this token can be used as the
// `ListDataSourcesRequest.page_token`
// to request the next page of list results.
string next_page_token = 2;
}
// A request to create a data transfer configuration. If new credentials are
// needed for this transfer configuration, an authorization code must be
// provided. If an authorization code is provided, the transfer configuration
// will be associated with the user id corresponding to the
// authorization code. Otherwise, the transfer configuration will be associated
// with the calling user.
message CreateTransferConfigRequest {
// The BigQuery project id where the transfer configuration should be created.
// Must be in the format projects/{project_id}/locations/{location_id}
// If specified location and location of the destination bigquery dataset
// do not match - the request will fail.
string parent = 1;
// Data transfer configuration to create.
TransferConfig transfer_config = 2;
// Optional OAuth2 authorization code to use with this transfer configuration.
// This is required if new credentials are needed, as indicated by
// `CheckValidCreds`.
// In order to obtain authorization_code, please make a
// request to
// https://www.gstatic.com/bigquerydatatransfer/oauthz/auth?client_id=<datatransferapiclientid>&scope=<data_source_scopes>&redirect_uri=<redirect_uri>
//
// * client_id should be OAuth client_id of BigQuery DTS API for the given
// data source returned by ListDataSources method.
// * data_source_scopes are the scopes returned by ListDataSources method.
// * redirect_uri is an optional parameter. If not specified, then
// authorization code is posted to the opener of authorization flow window.
// Otherwise it will be sent to the redirect uri. A special value of
// urn:ietf:wg:oauth:2.0:oob means that authorization code should be
// returned in the title bar of the browser, with the page text prompting
// the user to copy the code and paste it in the application.
string authorization_code = 3;
// Optional version info. If users want to find a very recent access token,
// that is, immediately after approving access, users have to set the
// version_info claim in the token request. To obtain the version_info, users
// must use the "none+gsession" response type. which be return a
// version_info back in the authorization response which be be put in a JWT
// claim in the token request.
string version_info = 5;
}
// A request to update a transfer configuration. To update the user id of the
// transfer configuration, an authorization code needs to be provided.
message UpdateTransferConfigRequest {
// Data transfer configuration to create.
TransferConfig transfer_config = 1;
// Optional OAuth2 authorization code to use with this transfer configuration.
// If it is provided, the transfer configuration will be associated with the
// authorizing user.
// In order to obtain authorization_code, please make a
// request to
// https://www.gstatic.com/bigquerydatatransfer/oauthz/auth?client_id=<datatransferapiclientid>&scope=<data_source_scopes>&redirect_uri=<redirect_uri>
//
// * client_id should be OAuth client_id of BigQuery DTS API for the given
// data source returned by ListDataSources method.
// * data_source_scopes are the scopes returned by ListDataSources method.
// * redirect_uri is an optional parameter. If not specified, then
// authorization code is posted to the opener of authorization flow window.
// Otherwise it will be sent to the redirect uri. A special value of
// urn:ietf:wg:oauth:2.0:oob means that authorization code should be
// returned in the title bar of the browser, with the page text prompting
// the user to copy the code and paste it in the application.
string authorization_code = 3;
// Required list of fields to be updated in this request.
google.protobuf.FieldMask update_mask = 4;
// Optional version info. If users want to find a very recent access token,
// that is, immediately after approving access, users have to set the
// version_info claim in the token request. To obtain the version_info, users
// must use the "none+gsession" response type. which be return a
// version_info back in the authorization response which be be put in a JWT
// claim in the token request.
string version_info = 5;
}
// A request to get data transfer information.
message GetTransferConfigRequest {
// The field will contain name of the resource requested, for example:
// `projects/{project_id}/transferConfigs/{config_id}`
string name = 1;
}
// A request to delete data transfer information. All associated transfer runs
// and log messages will be deleted as well.
message DeleteTransferConfigRequest {
// The field will contain name of the resource requested, for example:
// `projects/{project_id}/transferConfigs/{config_id}`
string name = 1;
}
// A request to get data transfer run information.
message GetTransferRunRequest {
// The field will contain name of the resource requested, for example:
// `projects/{project_id}/transferConfigs/{config_id}/runs/{run_id}`
string name = 1;
}
// A request to delete data transfer run information.
message DeleteTransferRunRequest {
// The field will contain name of the resource requested, for example:
// `projects/{project_id}/transferConfigs/{config_id}/runs/{run_id}`
string name = 1;
}
// A request to list data transfers configured for a BigQuery project.
message ListTransferConfigsRequest {
// The BigQuery project id for which data sources
// should be returned: `projects/{project_id}`.
string parent = 1;
// When specified, only configurations of requested data sources are returned.
repeated string data_source_ids = 2;
// Pagination token, which can be used to request a specific page
// of `ListTransfersRequest` list results. For multiple-page
// results, `ListTransfersResponse` outputs
// a `next_page` token, which can be used as the
// `page_token` value to request the next page of list results.
string page_token = 3;
// Page size. The default page size is the maximum value of 1000 results.
int32 page_size = 4;
}
// The returned list of pipelines in the project.
message ListTransferConfigsResponse {
// Output only. The stored pipeline transfer configurations.
repeated TransferConfig transfer_configs = 1;
// Output only. The next-pagination token. For multiple-page list results,
// this token can be used as the
// `ListTransferConfigsRequest.page_token`
// to request the next page of list results.
string next_page_token = 2;
}
// A request to list data transfer runs. UI can use this method to show/filter
// specific data transfer runs. The data source can use this method to request
// all scheduled transfer runs.
message ListTransferRunsRequest {
// Represents which runs should be pulled.
enum RunAttempt {
// All runs should be returned.
RUN_ATTEMPT_UNSPECIFIED = 0;
// Only latest run per day should be returned.
LATEST = 1;
}
// Name of transfer configuration for which transfer runs should be retrieved.
// Format of transfer configuration resource name is:
// `projects/{project_id}/transferConfigs/{config_id}`.
string parent = 1;
// When specified, only transfer runs with requested states are returned.
repeated TransferState states = 2;
// Pagination token, which can be used to request a specific page
// of `ListTransferRunsRequest` list results. For multiple-page
// results, `ListTransferRunsResponse` outputs
// a `next_page` token, which can be used as the
// `page_token` value to request the next page of list results.
string page_token = 3;
// Page size. The default page size is the maximum value of 1000 results.
int32 page_size = 4;
// Indicates how run attempts are to be pulled.
RunAttempt run_attempt = 5;
}
// The returned list of pipelines in the project.
message ListTransferRunsResponse {
// Output only. The stored pipeline transfer runs.
repeated TransferRun transfer_runs = 1;
// Output only. The next-pagination token. For multiple-page list results,
// this token can be used as the
// `ListTransferRunsRequest.page_token`
// to request the next page of list results.
string next_page_token = 2;
}
// A request to get user facing log messages associated with data transfer run.
message ListTransferLogsRequest {
// Transfer run name in the form:
// `projects/{project_id}/transferConfigs/{config_Id}/runs/{run_id}`.
string parent = 1;
// Pagination token, which can be used to request a specific page
// of `ListTransferLogsRequest` list results. For multiple-page
// results, `ListTransferLogsResponse` outputs
// a `next_page` token, which can be used as the
// `page_token` value to request the next page of list results.
string page_token = 4;
// Page size. The default page size is the maximum value of 1000 results.
int32 page_size = 5;
// Message types to return. If not populated - INFO, WARNING and ERROR
// messages are returned.
repeated TransferMessage.MessageSeverity message_types = 6;
}
// The returned list transfer run messages.
message ListTransferLogsResponse {
// Output only. The stored pipeline transfer messages.
repeated TransferMessage transfer_messages = 1;
// Output only. The next-pagination token. For multiple-page list results,
// this token can be used as the
// `GetTransferRunLogRequest.page_token`
// to request the next page of list results.
string next_page_token = 2;
}
// A request to determine whether the user has valid credentials. This method
// is used to limit the number of OAuth popups in the user interface. The
// user id is inferred from the API call context.
// If the data source has the Google+ authorization type, this method
// returns false, as it cannot be determined whether the credentials are
// already valid merely based on the user id.
message CheckValidCredsRequest {
// The data source in the form:
// `projects/{project_id}/dataSources/{data_source_id}`
string name = 1;
}
// A response indicating whether the credentials exist and are valid.
message CheckValidCredsResponse {
// If set to `true`, the credentials exist and are valid.
bool has_valid_creds = 1;
}
// A request to schedule transfer runs for a time range.
message ScheduleTransferRunsRequest {
// Transfer configuration name in the form:
// `projects/{project_id}/transferConfigs/{config_id}`.
string parent = 1;
// Start time of the range of transfer runs. For example,
// `"2017-05-25T00:00:00+00:00"`.
google.protobuf.Timestamp start_time = 2;
// End time of the range of transfer runs. For example,
// `"2017-05-30T00:00:00+00:00"`.
google.protobuf.Timestamp end_time = 3;
}
// A response to schedule transfer runs for a time range.
message ScheduleTransferRunsResponse {
// The transfer runs that were scheduled.
repeated TransferRun runs = 1;
}
// A request to start manual transfer runs.
message StartManualTransferRunsRequest {
// A specification for a time range, this will request transfer runs with
// run_time between start_time (inclusive) and end_time (exclusive).
message TimeRange {
// Start time of the range of transfer runs. For example,
// `"2017-05-25T00:00:00+00:00"`. The start_time must be strictly less than
// the end_time. Creates transfer runs where run_time is in the range betwen
// start_time (inclusive) and end_time (exlusive).
google.protobuf.Timestamp start_time = 1;
// End time of the range of transfer runs. For example,
// `"2017-05-30T00:00:00+00:00"`. The end_time must not be in the future.
// Creates transfer runs where run_time is in the range betwen start_time
// (inclusive) and end_time (exlusive).
google.protobuf.Timestamp end_time = 2;
}
// Transfer configuration name in the form:
// `projects/{project_id}/transferConfigs/{config_id}`.
string parent = 1;
// The requested time specification - this can be a time range or a specific
// run_time.
oneof time {
// Time range for the transfer runs that should be started.
TimeRange requested_time_range = 3;
// Specific run_time for a transfer run to be started. The
// requested_run_time must not be in the future.
google.protobuf.Timestamp requested_run_time = 4;
}
}
// A response to start manual transfer runs.
message StartManualTransferRunsResponse {
// The transfer runs that were created.
repeated TransferRun runs = 1;
}
You can’t perform that action at this time.