From 5b0893f015aa6ebb73ca34c629ef47ee2d5d325a Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 19 Feb 2020 19:58:09 +0000 Subject: [PATCH 1/7] feat: update synth to generate v1beta2, v1 endpoints for bigquerystorage There's also some work to try to accomodate v1alpha2 in this PR, but there exists further oddities that must be tackled before we can fully add generation of the alpha client here. Also, note that this PR does not include manual client modifications such as streaming offset resumption that are present in the v1beta1 client. Intent is to address further changes like that in subsequent PRs. --- docs/gapic/v1/api.rst | 6 + docs/gapic/v1/types.rst | 5 + docs/gapic/v1beta2/api.rst | 6 + docs/gapic/v1beta2/types.rst | 5 + google/cloud/bigquery_storage.py | 8 +- .../bigquery_storage_v1/gapic/__init__.py | 0 .../gapic/big_query_read_client.py | 459 ++++++++++ .../gapic/big_query_read_client_config.py | 57 ++ .../cloud/bigquery_storage_v1/gapic/enums.py | 36 + .../gapic/transports/__init__.py | 0 .../big_query_read_grpc_transport.py | 189 ++++ .../bigquery_storage_v1/proto/__init__.py | 0 .../bigquery_storage_v1/proto/arrow.proto | 45 + .../bigquery_storage_v1/proto/arrow_pb2.py | 174 ++++ .../proto/arrow_pb2_grpc.py | 2 + .../bigquery_storage_v1/proto/avro.proto | 41 + .../bigquery_storage_v1/proto/avro_pb2.py | 171 ++++ .../proto/avro_pb2_grpc.py | 2 + .../bigquery_storage_v1/proto/storage.proto | 236 +++++ .../bigquery_storage_v1/proto/storage_pb2.py | 867 ++++++++++++++++++ .../proto/storage_pb2_grpc.py | 125 +++ .../bigquery_storage_v1/proto/stream.proto | 138 +++ .../bigquery_storage_v1/proto/stream_pb2.py | 587 ++++++++++++ .../proto/stream_pb2_grpc.py | 2 + google/cloud/bigquery_storage_v1/types.py | 47 + .../gapic/__init__.py | 0 .../gapic/big_query_read_client.py | 459 ++++++++++ .../gapic/big_query_read_client_config.py | 57 ++ .../bigquery_storage_v1beta2/gapic/enums.py | 52 ++ .../gapic/transports/__init__.py | 0 .../big_query_read_grpc_transport.py | 189 ++++ .../proto/__init__.py | 0 .../proto/arrow.proto | 58 ++ .../proto/arrow_pb2.py | 246 +++++ .../proto/arrow_pb2_grpc.py | 2 + .../bigquery_storage_v1beta2/proto/avro.proto | 36 + .../proto/avro_pb2.py | 151 +++ .../proto/avro_pb2_grpc.py | 2 + .../proto/storage.proto | 230 +++++ .../proto/storage_pb2.py | 867 ++++++++++++++++++ .../proto/storage_pb2_grpc.py | 125 +++ .../proto/stream.proto | 143 +++ .../proto/stream_pb2.py | 615 +++++++++++++ .../proto/stream_pb2_grpc.py | 2 + .../cloud/bigquery_storage_v1beta2/types.py | 47 + synth.metadata | 26 +- synth.py | 60 +- .../gapic/v1/test_big_query_read_client_v1.py | 163 ++++ .../test_big_query_read_client_v1beta2.py | 163 ++++ 49 files changed, 6879 insertions(+), 22 deletions(-) create mode 100644 docs/gapic/v1/api.rst create mode 100644 docs/gapic/v1/types.rst create mode 100644 docs/gapic/v1beta2/api.rst create mode 100644 docs/gapic/v1beta2/types.rst create mode 100644 google/cloud/bigquery_storage_v1/gapic/__init__.py create mode 100644 google/cloud/bigquery_storage_v1/gapic/big_query_read_client.py create mode 100644 google/cloud/bigquery_storage_v1/gapic/big_query_read_client_config.py create mode 100644 google/cloud/bigquery_storage_v1/gapic/enums.py create mode 100644 google/cloud/bigquery_storage_v1/gapic/transports/__init__.py create mode 100644 google/cloud/bigquery_storage_v1/gapic/transports/big_query_read_grpc_transport.py create mode 100644 google/cloud/bigquery_storage_v1/proto/__init__.py create mode 100644 google/cloud/bigquery_storage_v1/proto/arrow.proto create mode 100644 google/cloud/bigquery_storage_v1/proto/arrow_pb2.py create mode 100644 google/cloud/bigquery_storage_v1/proto/arrow_pb2_grpc.py create mode 100644 google/cloud/bigquery_storage_v1/proto/avro.proto create mode 100644 google/cloud/bigquery_storage_v1/proto/avro_pb2.py create mode 100644 google/cloud/bigquery_storage_v1/proto/avro_pb2_grpc.py create mode 100644 google/cloud/bigquery_storage_v1/proto/storage.proto create mode 100644 google/cloud/bigquery_storage_v1/proto/storage_pb2.py create mode 100644 google/cloud/bigquery_storage_v1/proto/storage_pb2_grpc.py create mode 100644 google/cloud/bigquery_storage_v1/proto/stream.proto create mode 100644 google/cloud/bigquery_storage_v1/proto/stream_pb2.py create mode 100644 google/cloud/bigquery_storage_v1/proto/stream_pb2_grpc.py create mode 100644 google/cloud/bigquery_storage_v1/types.py create mode 100644 google/cloud/bigquery_storage_v1beta2/gapic/__init__.py create mode 100644 google/cloud/bigquery_storage_v1beta2/gapic/big_query_read_client.py create mode 100644 google/cloud/bigquery_storage_v1beta2/gapic/big_query_read_client_config.py create mode 100644 google/cloud/bigquery_storage_v1beta2/gapic/enums.py create mode 100644 google/cloud/bigquery_storage_v1beta2/gapic/transports/__init__.py create mode 100644 google/cloud/bigquery_storage_v1beta2/gapic/transports/big_query_read_grpc_transport.py create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/__init__.py create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/arrow.proto create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/arrow_pb2.py create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/arrow_pb2_grpc.py create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/avro.proto create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/avro_pb2.py create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/avro_pb2_grpc.py create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/storage.proto create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/storage_pb2.py create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/storage_pb2_grpc.py create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/stream.proto create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/stream_pb2.py create mode 100644 google/cloud/bigquery_storage_v1beta2/proto/stream_pb2_grpc.py create mode 100644 google/cloud/bigquery_storage_v1beta2/types.py create mode 100644 tests/unit/gapic/v1/test_big_query_read_client_v1.py create mode 100644 tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py diff --git a/docs/gapic/v1/api.rst b/docs/gapic/v1/api.rst new file mode 100644 index 00000000..f2d66087 --- /dev/null +++ b/docs/gapic/v1/api.rst @@ -0,0 +1,6 @@ +Client for BigQuery Storage API +=============================== + +.. automodule:: google.cloud.bigquery_storage_v1 + :members: + :inherited-members: \ No newline at end of file diff --git a/docs/gapic/v1/types.rst b/docs/gapic/v1/types.rst new file mode 100644 index 00000000..b2392e7a --- /dev/null +++ b/docs/gapic/v1/types.rst @@ -0,0 +1,5 @@ +Types for BigQuery Storage API Client +===================================== + +.. automodule:: google.cloud.bigquery_storage_v1.types + :members: \ No newline at end of file diff --git a/docs/gapic/v1beta2/api.rst b/docs/gapic/v1beta2/api.rst new file mode 100644 index 00000000..d97911ba --- /dev/null +++ b/docs/gapic/v1beta2/api.rst @@ -0,0 +1,6 @@ +Client for BigQuery Storage API +=============================== + +.. automodule:: google.cloud.bigquery_storage_v1beta2 + :members: + :inherited-members: \ No newline at end of file diff --git a/docs/gapic/v1beta2/types.rst b/docs/gapic/v1beta2/types.rst new file mode 100644 index 00000000..11711dca --- /dev/null +++ b/docs/gapic/v1beta2/types.rst @@ -0,0 +1,5 @@ +Types for BigQuery Storage API Client +===================================== + +.. automodule:: google.cloud.bigquery_storage_v1beta2.types + :members: \ No newline at end of file diff --git a/google/cloud/bigquery_storage.py b/google/cloud/bigquery_storage.py index f8179708..de61f2cb 100644 --- a/google/cloud/bigquery_storage.py +++ b/google/cloud/bigquery_storage.py @@ -17,9 +17,9 @@ from __future__ import absolute_import -from google.cloud.bigquery_storage_v1beta1 import BigQueryStorageClient -from google.cloud.bigquery_storage_v1beta1 import enums -from google.cloud.bigquery_storage_v1beta1 import types +from google.cloud.bigquery_storage_v1 import BigQueryReadClient +from google.cloud.bigquery_storage_v1 import enums +from google.cloud.bigquery_storage_v1 import types -__all__ = ("enums", "types", "BigQueryStorageClient") +__all__ = ("enums", "types", "BigQueryReadClient") diff --git a/google/cloud/bigquery_storage_v1/gapic/__init__.py b/google/cloud/bigquery_storage_v1/gapic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/google/cloud/bigquery_storage_v1/gapic/big_query_read_client.py b/google/cloud/bigquery_storage_v1/gapic/big_query_read_client.py new file mode 100644 index 00000000..4a96fca7 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/gapic/big_query_read_client.py @@ -0,0 +1,459 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Accesses the google.cloud.bigquery.storage.v1 BigQueryRead API.""" + +import pkg_resources +import warnings + +from google.oauth2 import service_account +import google.api_core.client_options +import google.api_core.gapic_v1.client_info +import google.api_core.gapic_v1.config +import google.api_core.gapic_v1.method +import google.api_core.path_template +import google.api_core.gapic_v1.routing_header +import google.api_core.grpc_helpers +import grpc + +from google.cloud.bigquery_storage_v1.gapic import big_query_read_client_config +from google.cloud.bigquery_storage_v1.gapic import enums +from google.cloud.bigquery_storage_v1.gapic.transports import ( + big_query_read_grpc_transport, +) +from google.cloud.bigquery_storage_v1.proto import storage_pb2 +from google.cloud.bigquery_storage_v1.proto import storage_pb2_grpc +from google.cloud.bigquery_storage_v1.proto import stream_pb2 + + +_GAPIC_LIBRARY_VERSION = pkg_resources.get_distribution( + "google-cloud-bigquery-storage" +).version + + +class BigQueryReadClient(object): + """ + BigQuery Read API. + + The Read API can be used to read data from BigQuery. + """ + + SERVICE_ADDRESS = "bigquerystorage.googleapis.com:443" + """The default address of the service.""" + + # The name of the interface for this client. This is the key used to + # find the method configuration in the client_config dictionary. + _INTERFACE_NAME = "google.cloud.bigquery.storage.v1.BigQueryRead" + + @classmethod + def from_service_account_file(cls, filename, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file(filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + def __init__( + self, + transport=None, + channel=None, + credentials=None, + client_config=None, + client_info=None, + client_options=None, + ): + """Constructor. + + Args: + transport (Union[~.BigQueryReadGrpcTransport, + Callable[[~.Credentials, type], ~.BigQueryReadGrpcTransport]): A transport + instance, responsible for actually making the API calls. + The default transport uses the gRPC protocol. + This argument may also be a callable which returns a + transport instance. Callables will be sent the credentials + as the first argument and the default transport class as + the second argument. + channel (grpc.Channel): DEPRECATED. A ``Channel`` instance + through which to make calls. This argument is mutually exclusive + with ``credentials``; providing both will raise an exception. + credentials (google.auth.credentials.Credentials): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is mutually exclusive with providing a + transport instance to ``transport``; doing so will raise + an exception. + client_config (dict): DEPRECATED. A dictionary of call options for + each method. If not specified, the default configuration is used. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + client_options (Union[dict, google.api_core.client_options.ClientOptions]): + Client options used to set user options on the client. API Endpoint + should be set through client_options. + """ + # Raise deprecation warnings for things we want to go away. + if client_config is not None: + warnings.warn( + "The `client_config` argument is deprecated.", + PendingDeprecationWarning, + stacklevel=2, + ) + else: + client_config = big_query_read_client_config.config + + if channel: + warnings.warn( + "The `channel` argument is deprecated; use " "`transport` instead.", + PendingDeprecationWarning, + stacklevel=2, + ) + + api_endpoint = self.SERVICE_ADDRESS + if client_options: + if type(client_options) == dict: + client_options = google.api_core.client_options.from_dict( + client_options + ) + if client_options.api_endpoint: + api_endpoint = client_options.api_endpoint + + # Instantiate the transport. + # The transport is responsible for handling serialization and + # deserialization and actually sending data to the service. + if transport: # pragma: no cover + if callable(transport): + self.transport = transport( + credentials=credentials, + default_class=big_query_read_grpc_transport.BigQueryReadGrpcTransport, + address=api_endpoint, + ) + else: + if credentials: + raise ValueError( + "Received both a transport instance and " + "credentials; these are mutually exclusive." + ) + self.transport = transport + else: + self.transport = big_query_read_grpc_transport.BigQueryReadGrpcTransport( + address=api_endpoint, channel=channel, credentials=credentials + ) + + if client_info is None: + client_info = google.api_core.gapic_v1.client_info.ClientInfo( + gapic_version=_GAPIC_LIBRARY_VERSION + ) + else: + client_info.gapic_version = _GAPIC_LIBRARY_VERSION + self._client_info = client_info + + # Parse out the default settings for retry and timeout for each RPC + # from the client configuration. + # (Ordinarily, these are the defaults specified in the `*_config.py` + # file next to this one.) + self._method_configs = google.api_core.gapic_v1.config.parse_method_configs( + client_config["interfaces"][self._INTERFACE_NAME] + ) + + # Save a dictionary of cached API call functions. + # These are the actual callables which invoke the proper + # transport methods, wrapped with `wrap_method` to add retry, + # timeout, and the like. + self._inner_api_calls = {} + + # Service calls + def create_read_session( + self, + parent=None, + read_session=None, + max_stream_count=None, + retry=google.api_core.gapic_v1.method.DEFAULT, + timeout=google.api_core.gapic_v1.method.DEFAULT, + metadata=None, + ): + """ + Creates a new read session. A read session divides the contents of a + BigQuery table into one or more streams, which can then be used to read + data from the table. The read session also specifies properties of the + data to be read, such as a list of columns or a push-down filter describing + the rows to be returned. + + A particular row can be read by at most one stream. When the caller has + reached the end of each stream in the session, then all the data in the + table has been read. + + Data is assigned to each stream such that roughly the same number of + rows can be read from each stream. Because the server-side unit for + assigning data is collections of rows, the API does not guarantee that + each stream will return the same number or rows. Additionally, the + limits are enforced based on the number of pre-filtered rows, so some + filters can lead to lopsided assignments. + + Read sessions automatically expire 24 hours after they are created and do + not require manual clean-up by the caller. + + Example: + >>> from google.cloud import bigquery_storage_v1 + >>> + >>> client = bigquery_storage_v1.BigQueryReadClient() + >>> + >>> response = client.create_read_session() + + Args: + parent (str): Required. The request project that owns the session, in the form of + ``projects/{project_id}``. + read_session (Union[dict, ~google.cloud.bigquery_storage_v1.types.ReadSession]): Required. Session to be created. + + If a dict is provided, it must be of the same form as the protobuf + message :class:`~google.cloud.bigquery_storage_v1.types.ReadSession` + max_stream_count (int): Max initial number of streams. If unset or zero, the server will + provide a value of streams so as to produce reasonable throughput. Must be + non-negative. The number of streams may be lower than the requested number, + depending on the amount parallelism that is reasonable for the table. Error + will be returned if the max count is greater than the current system + max limit of 1,000. + + Streams must be read starting from offset 0. + retry (Optional[google.api_core.retry.Retry]): A retry object used + to retry requests. If ``None`` is specified, requests will + be retried using a default configuration. + timeout (Optional[float]): The amount of time, in seconds, to wait + for the request to complete. Note that if ``retry`` is + specified, the timeout applies to each individual attempt. + metadata (Optional[Sequence[Tuple[str, str]]]): Additional metadata + that is provided to the method. + + Returns: + A :class:`~google.cloud.bigquery_storage_v1.types.ReadSession` instance. + + Raises: + google.api_core.exceptions.GoogleAPICallError: If the request + failed for any reason. + google.api_core.exceptions.RetryError: If the request failed due + to a retryable error and retry attempts failed. + ValueError: If the parameters are invalid. + """ + # Wrap the transport method to add retry and timeout logic. + if "create_read_session" not in self._inner_api_calls: + self._inner_api_calls[ + "create_read_session" + ] = google.api_core.gapic_v1.method.wrap_method( + self.transport.create_read_session, + default_retry=self._method_configs["CreateReadSession"].retry, + default_timeout=self._method_configs["CreateReadSession"].timeout, + client_info=self._client_info, + ) + + request = storage_pb2.CreateReadSessionRequest( + parent=parent, read_session=read_session, max_stream_count=max_stream_count + ) + if metadata is None: + metadata = [] + metadata = list(metadata) + try: + routing_header = [("read_session.table", read_session.table)] + except AttributeError: + pass + else: + routing_metadata = google.api_core.gapic_v1.routing_header.to_grpc_metadata( + routing_header + ) + metadata.append(routing_metadata) # pragma: no cover + + return self._inner_api_calls["create_read_session"]( + request, retry=retry, timeout=timeout, metadata=metadata + ) + + def read_rows( + self, + read_stream=None, + offset=None, + retry=google.api_core.gapic_v1.method.DEFAULT, + timeout=google.api_core.gapic_v1.method.DEFAULT, + metadata=None, + ): + """ + Reads rows from the stream in the format prescribed by the ReadSession. + Each response contains one or more table rows, up to a maximum of 100 MiB + per response; read requests which attempt to read individual rows larger + than 100 MiB will fail. + + Each request also returns a set of stream statistics reflecting the current + state of the stream. + + Example: + >>> from google.cloud import bigquery_storage_v1 + >>> + >>> client = bigquery_storage_v1.BigQueryReadClient() + >>> + >>> for element in client.read_rows(): + ... # process element + ... pass + + Args: + read_stream (str): Required. Stream to read rows from. + offset (long): The offset requested must be less than the last row read from Read. + Requesting a larger offset is undefined. If not specified, start reading + from offset zero. + retry (Optional[google.api_core.retry.Retry]): A retry object used + to retry requests. If ``None`` is specified, requests will + be retried using a default configuration. + timeout (Optional[float]): The amount of time, in seconds, to wait + for the request to complete. Note that if ``retry`` is + specified, the timeout applies to each individual attempt. + metadata (Optional[Sequence[Tuple[str, str]]]): Additional metadata + that is provided to the method. + + Returns: + Iterable[~google.cloud.bigquery_storage_v1.types.ReadRowsResponse]. + + Raises: + google.api_core.exceptions.GoogleAPICallError: If the request + failed for any reason. + google.api_core.exceptions.RetryError: If the request failed due + to a retryable error and retry attempts failed. + ValueError: If the parameters are invalid. + """ + # Wrap the transport method to add retry and timeout logic. + if "read_rows" not in self._inner_api_calls: + self._inner_api_calls[ + "read_rows" + ] = google.api_core.gapic_v1.method.wrap_method( + self.transport.read_rows, + default_retry=self._method_configs["ReadRows"].retry, + default_timeout=self._method_configs["ReadRows"].timeout, + client_info=self._client_info, + ) + + request = storage_pb2.ReadRowsRequest(read_stream=read_stream, offset=offset) + if metadata is None: + metadata = [] + metadata = list(metadata) + try: + routing_header = [("read_stream", read_stream)] + except AttributeError: + pass + else: + routing_metadata = google.api_core.gapic_v1.routing_header.to_grpc_metadata( + routing_header + ) + metadata.append(routing_metadata) # pragma: no cover + + return self._inner_api_calls["read_rows"]( + request, retry=retry, timeout=timeout, metadata=metadata + ) + + def split_read_stream( + self, + name=None, + fraction=None, + retry=google.api_core.gapic_v1.method.DEFAULT, + timeout=google.api_core.gapic_v1.method.DEFAULT, + metadata=None, + ): + """ + Splits a given ``ReadStream`` into two ``ReadStream`` objects. These + ``ReadStream`` objects are referred to as the primary and the residual + streams of the split. The original ``ReadStream`` can still be read from + in the same manner as before. Both of the returned ``ReadStream`` + objects can also be read from, and the rows returned by both child + streams will be the same as the rows read from the original stream. + + Moreover, the two child streams will be allocated back-to-back in the + original ``ReadStream``. Concretely, it is guaranteed that for streams + original, primary, and residual, that original[0-j] = primary[0-j] and + original[j-n] = residual[0-m] once the streams have been read to + completion. + + Example: + >>> from google.cloud import bigquery_storage_v1 + >>> + >>> client = bigquery_storage_v1.BigQueryReadClient() + >>> + >>> response = client.split_read_stream() + + Args: + name (str): Required. Name of the stream to split. + fraction (float): A value in the range (0.0, 1.0) that specifies the fractional point at + which the original stream should be split. The actual split point is + evaluated on pre-filtered rows, so if a filter is provided, then there is + no guarantee that the division of the rows between the new child streams + will be proportional to this fractional value. Additionally, because the + server-side unit for assigning data is collections of rows, this fraction + will always map to a data storage boundary on the server side. + retry (Optional[google.api_core.retry.Retry]): A retry object used + to retry requests. If ``None`` is specified, requests will + be retried using a default configuration. + timeout (Optional[float]): The amount of time, in seconds, to wait + for the request to complete. Note that if ``retry`` is + specified, the timeout applies to each individual attempt. + metadata (Optional[Sequence[Tuple[str, str]]]): Additional metadata + that is provided to the method. + + Returns: + A :class:`~google.cloud.bigquery_storage_v1.types.SplitReadStreamResponse` instance. + + Raises: + google.api_core.exceptions.GoogleAPICallError: If the request + failed for any reason. + google.api_core.exceptions.RetryError: If the request failed due + to a retryable error and retry attempts failed. + ValueError: If the parameters are invalid. + """ + # Wrap the transport method to add retry and timeout logic. + if "split_read_stream" not in self._inner_api_calls: + self._inner_api_calls[ + "split_read_stream" + ] = google.api_core.gapic_v1.method.wrap_method( + self.transport.split_read_stream, + default_retry=self._method_configs["SplitReadStream"].retry, + default_timeout=self._method_configs["SplitReadStream"].timeout, + client_info=self._client_info, + ) + + request = storage_pb2.SplitReadStreamRequest(name=name, fraction=fraction) + if metadata is None: + metadata = [] + metadata = list(metadata) + try: + routing_header = [("name", name)] + except AttributeError: + pass + else: + routing_metadata = google.api_core.gapic_v1.routing_header.to_grpc_metadata( + routing_header + ) + metadata.append(routing_metadata) # pragma: no cover + + return self._inner_api_calls["split_read_stream"]( + request, retry=retry, timeout=timeout, metadata=metadata + ) diff --git a/google/cloud/bigquery_storage_v1/gapic/big_query_read_client_config.py b/google/cloud/bigquery_storage_v1/gapic/big_query_read_client_config.py new file mode 100644 index 00000000..cac22843 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/gapic/big_query_read_client_config.py @@ -0,0 +1,57 @@ +config = { + "interfaces": { + "google.cloud.bigquery.storage.v1.BigQueryRead": { + "retry_codes": { + "idempotent": ["DEADLINE_EXCEEDED", "UNAVAILABLE"], + "unary_streaming": ["UNAVAILABLE"], + "non_idempotent": [], + }, + "retry_params": { + "default": { + "initial_retry_delay_millis": 100, + "retry_delay_multiplier": 1.3, + "max_retry_delay_millis": 60000, + "initial_rpc_timeout_millis": 20000, + "rpc_timeout_multiplier": 1.0, + "max_rpc_timeout_millis": 20000, + "total_timeout_millis": 600000, + }, + "create_read_session": { + "initial_retry_delay_millis": 100, + "retry_delay_multiplier": 1.3, + "max_retry_delay_millis": 60000, + "initial_rpc_timeout_millis": 120000, + "rpc_timeout_multiplier": 1.0, + "max_rpc_timeout_millis": 120000, + "total_timeout_millis": 600000, + }, + "read_rows": { + "initial_retry_delay_millis": 100, + "retry_delay_multiplier": 1.3, + "max_retry_delay_millis": 60000, + "initial_rpc_timeout_millis": 86400000, + "rpc_timeout_multiplier": 1.0, + "max_rpc_timeout_millis": 86400000, + "total_timeout_millis": 86400000, + }, + }, + "methods": { + "CreateReadSession": { + "timeout_millis": 120000, + "retry_codes_name": "idempotent", + "retry_params_name": "create_read_session", + }, + "ReadRows": { + "timeout_millis": 21600000, + "retry_codes_name": "unary_streaming", + "retry_params_name": "read_rows", + }, + "SplitReadStream": { + "timeout_millis": 120000, + "retry_codes_name": "idempotent", + "retry_params_name": "default", + }, + }, + } + } +} diff --git a/google/cloud/bigquery_storage_v1/gapic/enums.py b/google/cloud/bigquery_storage_v1/gapic/enums.py new file mode 100644 index 00000000..effa16bd --- /dev/null +++ b/google/cloud/bigquery_storage_v1/gapic/enums.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Wrappers for protocol buffer enum types.""" + +import enum + + +class DataFormat(enum.IntEnum): + """ + Data format for input or output data. + + Attributes: + DATA_FORMAT_UNSPECIFIED (int) + AVRO (int): Avro is a standard open source row based file format. + See https://avro.apache.org/ for more details. + ARROW (int): Arrow is a standard open source column-based message format. + See https://arrow.apache.org/ for more details. + """ + + DATA_FORMAT_UNSPECIFIED = 0 + AVRO = 1 + ARROW = 2 diff --git a/google/cloud/bigquery_storage_v1/gapic/transports/__init__.py b/google/cloud/bigquery_storage_v1/gapic/transports/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/google/cloud/bigquery_storage_v1/gapic/transports/big_query_read_grpc_transport.py b/google/cloud/bigquery_storage_v1/gapic/transports/big_query_read_grpc_transport.py new file mode 100644 index 00000000..2223b487 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/gapic/transports/big_query_read_grpc_transport.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import google.api_core.grpc_helpers + +from google.cloud.bigquery_storage_v1.proto import storage_pb2_grpc + + +class BigQueryReadGrpcTransport(object): + """gRPC transport class providing stubs for + google.cloud.bigquery.storage.v1 BigQueryRead API. + + The transport provides access to the raw gRPC stubs, + which can be used to take advantage of advanced + features of gRPC. + """ + + # The scopes needed to make gRPC calls to all of the methods defined + # in this service. + _OAUTH_SCOPES = ( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.readonly", + "https://www.googleapis.com/auth/cloud-platform", + ) + + def __init__( + self, + channel=None, + credentials=None, + address="bigquerystorage.googleapis.com:443", + ): + """Instantiate the transport class. + + Args: + channel (grpc.Channel): A ``Channel`` instance through + which to make calls. This argument is mutually exclusive + with ``credentials``; providing both will raise an exception. + credentials (google.auth.credentials.Credentials): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If none + are specified, the client will attempt to ascertain the + credentials from the environment. + address (str): The address where the service is hosted. + """ + # If both `channel` and `credentials` are specified, raise an + # exception (channels come with credentials baked in already). + if channel is not None and credentials is not None: # pragma: no cover + raise ValueError( + "The `channel` and `credentials` arguments are mutually " "exclusive." + ) + + # Create the channel. + if channel is None: # pragma: no cover + channel = self.create_channel( + address=address, + credentials=credentials, + options={ + "grpc.max_send_message_length": -1, + "grpc.max_receive_message_length": -1, + }.items(), + ) + + self._channel = channel + + # gRPC uses objects called "stubs" that are bound to the + # channel and provide a basic method for each RPC. + self._stubs = { + "big_query_read_stub": storage_pb2_grpc.BigQueryReadStub(channel) + } + + @classmethod + def create_channel( + cls, address="bigquerystorage.googleapis.com:443", credentials=None, **kwargs + ): + """Create and return a gRPC channel object. + + Args: + address (str): The host for the channel to use. + credentials (~.Credentials): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + kwargs (dict): Keyword arguments, which are passed to the + channel creation. + + Returns: + grpc.Channel: A gRPC channel object. + """ + return google.api_core.grpc_helpers.create_channel( # pragma: no cover + address, credentials=credentials, scopes=cls._OAUTH_SCOPES, **kwargs + ) + + @property + def channel(self): + """The gRPC channel used by the transport. + + Returns: + grpc.Channel: A gRPC channel object. + """ + return self._channel + + @property + def create_read_session(self): + """Return the gRPC stub for :meth:`BigQueryReadClient.create_read_session`. + + Creates a new read session. A read session divides the contents of a + BigQuery table into one or more streams, which can then be used to read + data from the table. The read session also specifies properties of the + data to be read, such as a list of columns or a push-down filter describing + the rows to be returned. + + A particular row can be read by at most one stream. When the caller has + reached the end of each stream in the session, then all the data in the + table has been read. + + Data is assigned to each stream such that roughly the same number of + rows can be read from each stream. Because the server-side unit for + assigning data is collections of rows, the API does not guarantee that + each stream will return the same number or rows. Additionally, the + limits are enforced based on the number of pre-filtered rows, so some + filters can lead to lopsided assignments. + + Read sessions automatically expire 24 hours after they are created and do + not require manual clean-up by the caller. + + Returns: + Callable: A callable which accepts the appropriate + deserialized request object and returns a + deserialized response object. + """ + return self._stubs["big_query_read_stub"].CreateReadSession + + @property + def read_rows(self): + """Return the gRPC stub for :meth:`BigQueryReadClient.read_rows`. + + Reads rows from the stream in the format prescribed by the ReadSession. + Each response contains one or more table rows, up to a maximum of 100 MiB + per response; read requests which attempt to read individual rows larger + than 100 MiB will fail. + + Each request also returns a set of stream statistics reflecting the current + state of the stream. + + Returns: + Callable: A callable which accepts the appropriate + deserialized request object and returns a + deserialized response object. + """ + return self._stubs["big_query_read_stub"].ReadRows + + @property + def split_read_stream(self): + """Return the gRPC stub for :meth:`BigQueryReadClient.split_read_stream`. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. These + ``ReadStream`` objects are referred to as the primary and the residual + streams of the split. The original ``ReadStream`` can still be read from + in the same manner as before. Both of the returned ``ReadStream`` + objects can also be read from, and the rows returned by both child + streams will be the same as the rows read from the original stream. + + Moreover, the two child streams will be allocated back-to-back in the + original ``ReadStream``. Concretely, it is guaranteed that for streams + original, primary, and residual, that original[0-j] = primary[0-j] and + original[j-n] = residual[0-m] once the streams have been read to + completion. + + Returns: + Callable: A callable which accepts the appropriate + deserialized request object and returns a + deserialized response object. + """ + return self._stubs["big_query_read_stub"].SplitReadStream diff --git a/google/cloud/bigquery_storage_v1/proto/__init__.py b/google/cloud/bigquery_storage_v1/proto/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/google/cloud/bigquery_storage_v1/proto/arrow.proto b/google/cloud/bigquery_storage_v1/proto/arrow.proto new file mode 100644 index 00000000..90add978 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/arrow.proto @@ -0,0 +1,45 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.cloud.bigquery.storage.v1; + +option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1"; +option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage"; +option java_multiple_files = true; +option java_outer_classname = "ArrowProto"; +option java_package = "com.google.cloud.bigquery.storage.v1"; +option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1"; + +// Arrow schema as specified in +// https://arrow.apache.org/docs/python/api/datatypes.html +// and serialized to bytes using IPC: +// https://arrow.apache.org/docs/ipc.html. +// +// See code samples on how this message can be deserialized. +message ArrowSchema { + // IPC serialized Arrow schema. + bytes serialized_schema = 1; +} + +// Arrow RecordBatch. +message ArrowRecordBatch { + // IPC-serialized Arrow RecordBatch. + bytes serialized_record_batch = 1; + + // The count of rows in `serialized_record_batch`. + int64 row_count = 2; +} diff --git a/google/cloud/bigquery_storage_v1/proto/arrow_pb2.py b/google/cloud/bigquery_storage_v1/proto/arrow_pb2.py new file mode 100644 index 00000000..d3570dea --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/arrow_pb2.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: google/cloud/bigquery/storage_v1/proto/arrow.proto + +import sys + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="google/cloud/bigquery/storage_v1/proto/arrow.proto", + package="google.cloud.bigquery.storage.v1", + syntax="proto3", + serialized_options=_b( + "\n$com.google.cloud.bigquery.storage.v1B\nArrowProtoP\001ZGgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage\252\002 Google.Cloud.BigQuery.Storage.V1\312\002 Google\\Cloud\\BigQuery\\Storage\\V1" + ), + serialized_pb=_b( + '\n2google/cloud/bigquery/storage_v1/proto/arrow.proto\x12 google.cloud.bigquery.storage.v1"(\n\x0b\x41rrowSchema\x12\x19\n\x11serialized_schema\x18\x01 \x01(\x0c"F\n\x10\x41rrowRecordBatch\x12\x1f\n\x17serialized_record_batch\x18\x01 \x01(\x0c\x12\x11\n\trow_count\x18\x02 \x01(\x03\x42\xc3\x01\n$com.google.cloud.bigquery.storage.v1B\nArrowProtoP\x01ZGgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage\xaa\x02 Google.Cloud.BigQuery.Storage.V1\xca\x02 Google\\Cloud\\BigQuery\\Storage\\V1b\x06proto3' + ), +) + + +_ARROWSCHEMA = _descriptor.Descriptor( + name="ArrowSchema", + full_name="google.cloud.bigquery.storage.v1.ArrowSchema", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="serialized_schema", + full_name="google.cloud.bigquery.storage.v1.ArrowSchema.serialized_schema", + index=0, + number=1, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=88, + serialized_end=128, +) + + +_ARROWRECORDBATCH = _descriptor.Descriptor( + name="ArrowRecordBatch", + full_name="google.cloud.bigquery.storage.v1.ArrowRecordBatch", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="serialized_record_batch", + full_name="google.cloud.bigquery.storage.v1.ArrowRecordBatch.serialized_record_batch", + index=0, + number=1, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="row_count", + full_name="google.cloud.bigquery.storage.v1.ArrowRecordBatch.row_count", + index=1, + number=2, + type=3, + cpp_type=2, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=130, + serialized_end=200, +) + +DESCRIPTOR.message_types_by_name["ArrowSchema"] = _ARROWSCHEMA +DESCRIPTOR.message_types_by_name["ArrowRecordBatch"] = _ARROWRECORDBATCH +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +ArrowSchema = _reflection.GeneratedProtocolMessageType( + "ArrowSchema", + (_message.Message,), + dict( + DESCRIPTOR=_ARROWSCHEMA, + __module__="google.cloud.bigquery.storage_v1.proto.arrow_pb2", + __doc__="""Arrow schema as specified in + https://arrow.apache.org/docs/python/api/datatypes.html and serialized + to bytes using IPC: https://arrow.apache.org/docs/ipc.html. + + See code samples on how this message can be deserialized. + + + Attributes: + serialized_schema: + IPC serialized Arrow schema. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.ArrowSchema) + ), +) +_sym_db.RegisterMessage(ArrowSchema) + +ArrowRecordBatch = _reflection.GeneratedProtocolMessageType( + "ArrowRecordBatch", + (_message.Message,), + dict( + DESCRIPTOR=_ARROWRECORDBATCH, + __module__="google.cloud.bigquery.storage_v1.proto.arrow_pb2", + __doc__="""Arrow RecordBatch. + + + Attributes: + serialized_record_batch: + IPC-serialized Arrow RecordBatch. + row_count: + The count of rows in ``serialized_record_batch``. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.ArrowRecordBatch) + ), +) +_sym_db.RegisterMessage(ArrowRecordBatch) + + +DESCRIPTOR._options = None +# @@protoc_insertion_point(module_scope) diff --git a/google/cloud/bigquery_storage_v1/proto/arrow_pb2_grpc.py b/google/cloud/bigquery_storage_v1/proto/arrow_pb2_grpc.py new file mode 100644 index 00000000..07cb78fe --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/arrow_pb2_grpc.py @@ -0,0 +1,2 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc diff --git a/google/cloud/bigquery_storage_v1/proto/avro.proto b/google/cloud/bigquery_storage_v1/proto/avro.proto new file mode 100644 index 00000000..9a064447 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/avro.proto @@ -0,0 +1,41 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.cloud.bigquery.storage.v1; + +option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1"; +option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage"; +option java_multiple_files = true; +option java_outer_classname = "AvroProto"; +option java_package = "com.google.cloud.bigquery.storage.v1"; +option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1"; + +// Avro schema. +message AvroSchema { + // Json serialized schema, as described at + // https://avro.apache.org/docs/1.8.1/spec.html. + string schema = 1; +} + +// Avro rows. +message AvroRows { + // Binary serialized rows in a block. + bytes serialized_binary_rows = 1; + + // The count of rows in the returning block. + int64 row_count = 2; +} diff --git a/google/cloud/bigquery_storage_v1/proto/avro_pb2.py b/google/cloud/bigquery_storage_v1/proto/avro_pb2.py new file mode 100644 index 00000000..c824282c --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/avro_pb2.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: google/cloud/bigquery/storage_v1/proto/avro.proto + +import sys + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="google/cloud/bigquery/storage_v1/proto/avro.proto", + package="google.cloud.bigquery.storage.v1", + syntax="proto3", + serialized_options=_b( + "\n$com.google.cloud.bigquery.storage.v1B\tAvroProtoP\001ZGgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage\252\002 Google.Cloud.BigQuery.Storage.V1\312\002 Google\\Cloud\\BigQuery\\Storage\\V1" + ), + serialized_pb=_b( + '\n1google/cloud/bigquery/storage_v1/proto/avro.proto\x12 google.cloud.bigquery.storage.v1"\x1c\n\nAvroSchema\x12\x0e\n\x06schema\x18\x01 \x01(\t"=\n\x08\x41vroRows\x12\x1e\n\x16serialized_binary_rows\x18\x01 \x01(\x0c\x12\x11\n\trow_count\x18\x02 \x01(\x03\x42\xc2\x01\n$com.google.cloud.bigquery.storage.v1B\tAvroProtoP\x01ZGgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage\xaa\x02 Google.Cloud.BigQuery.Storage.V1\xca\x02 Google\\Cloud\\BigQuery\\Storage\\V1b\x06proto3' + ), +) + + +_AVROSCHEMA = _descriptor.Descriptor( + name="AvroSchema", + full_name="google.cloud.bigquery.storage.v1.AvroSchema", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="schema", + full_name="google.cloud.bigquery.storage.v1.AvroSchema.schema", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=87, + serialized_end=115, +) + + +_AVROROWS = _descriptor.Descriptor( + name="AvroRows", + full_name="google.cloud.bigquery.storage.v1.AvroRows", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="serialized_binary_rows", + full_name="google.cloud.bigquery.storage.v1.AvroRows.serialized_binary_rows", + index=0, + number=1, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="row_count", + full_name="google.cloud.bigquery.storage.v1.AvroRows.row_count", + index=1, + number=2, + type=3, + cpp_type=2, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=117, + serialized_end=178, +) + +DESCRIPTOR.message_types_by_name["AvroSchema"] = _AVROSCHEMA +DESCRIPTOR.message_types_by_name["AvroRows"] = _AVROROWS +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +AvroSchema = _reflection.GeneratedProtocolMessageType( + "AvroSchema", + (_message.Message,), + dict( + DESCRIPTOR=_AVROSCHEMA, + __module__="google.cloud.bigquery.storage_v1.proto.avro_pb2", + __doc__="""Avro schema. + + + Attributes: + schema: + Json serialized schema, as described at + https://avro.apache.org/docs/1.8.1/spec.html. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.AvroSchema) + ), +) +_sym_db.RegisterMessage(AvroSchema) + +AvroRows = _reflection.GeneratedProtocolMessageType( + "AvroRows", + (_message.Message,), + dict( + DESCRIPTOR=_AVROROWS, + __module__="google.cloud.bigquery.storage_v1.proto.avro_pb2", + __doc__="""Avro rows. + + + Attributes: + serialized_binary_rows: + Binary serialized rows in a block. + row_count: + The count of rows in the returning block. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.AvroRows) + ), +) +_sym_db.RegisterMessage(AvroRows) + + +DESCRIPTOR._options = None +# @@protoc_insertion_point(module_scope) diff --git a/google/cloud/bigquery_storage_v1/proto/avro_pb2_grpc.py b/google/cloud/bigquery_storage_v1/proto/avro_pb2_grpc.py new file mode 100644 index 00000000..07cb78fe --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/avro_pb2_grpc.py @@ -0,0 +1,2 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc diff --git a/google/cloud/bigquery_storage_v1/proto/storage.proto b/google/cloud/bigquery_storage_v1/proto/storage.proto new file mode 100644 index 00000000..26fcd6ac --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/storage.proto @@ -0,0 +1,236 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.cloud.bigquery.storage.v1; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/cloud/bigquery/storage/v1/arrow.proto"; +import "google/cloud/bigquery/storage/v1/avro.proto"; +import "google/cloud/bigquery/storage/v1/stream.proto"; + +option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1"; +option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage"; +option java_multiple_files = true; +option java_outer_classname = "StorageProto"; +option java_package = "com.google.cloud.bigquery.storage.v1"; +option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1"; +option (google.api.resource_definition) = { + type: "bigquery.googleapis.com/Table" + pattern: "projects/{project}/datasets/{dataset}/tables/{table}" +}; + +// BigQuery Read API. +// +// The Read API can be used to read data from BigQuery. +service BigQueryRead { + option (google.api.default_host) = "bigquerystorage.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/bigquery," + "https://www.googleapis.com/auth/bigquery.readonly," + "https://www.googleapis.com/auth/cloud-platform"; + + // Creates a new read session. A read session divides the contents of a + // BigQuery table into one or more streams, which can then be used to read + // data from the table. The read session also specifies properties of the + // data to be read, such as a list of columns or a push-down filter describing + // the rows to be returned. + // + // A particular row can be read by at most one stream. When the caller has + // reached the end of each stream in the session, then all the data in the + // table has been read. + // + // Data is assigned to each stream such that roughly the same number of + // rows can be read from each stream. Because the server-side unit for + // assigning data is collections of rows, the API does not guarantee that + // each stream will return the same number or rows. Additionally, the + // limits are enforced based on the number of pre-filtered rows, so some + // filters can lead to lopsided assignments. + // + // Read sessions automatically expire 24 hours after they are created and do + // not require manual clean-up by the caller. + rpc CreateReadSession(CreateReadSessionRequest) returns (ReadSession) { + option (google.api.http) = { + post: "/v1/{read_session.table=projects/*/datasets/*/tables/*}" + body: "*" + }; + option (google.api.method_signature) = "parent,read_session,max_stream_count"; + } + + // Reads rows from the stream in the format prescribed by the ReadSession. + // Each response contains one or more table rows, up to a maximum of 100 MiB + // per response; read requests which attempt to read individual rows larger + // than 100 MiB will fail. + // + // Each request also returns a set of stream statistics reflecting the current + // state of the stream. + rpc ReadRows(ReadRowsRequest) returns (stream ReadRowsResponse) { + option (google.api.http) = { + get: "/v1/{read_stream=projects/*/locations/*/sessions/*/streams/*}" + }; + option (google.api.method_signature) = "read_stream,offset"; + } + + // Splits a given `ReadStream` into two `ReadStream` objects. These + // `ReadStream` objects are referred to as the primary and the residual + // streams of the split. The original `ReadStream` can still be read from in + // the same manner as before. Both of the returned `ReadStream` objects can + // also be read from, and the rows returned by both child streams will be + // the same as the rows read from the original stream. + // + // Moreover, the two child streams will be allocated back-to-back in the + // original `ReadStream`. Concretely, it is guaranteed that for streams + // original, primary, and residual, that original[0-j] = primary[0-j] and + // original[j-n] = residual[0-m] once the streams have been read to + // completion. + rpc SplitReadStream(SplitReadStreamRequest) returns (SplitReadStreamResponse) { + option (google.api.http) = { + get: "/v1/{name=projects/*/locations/*/sessions/*/streams/*}" + }; + } +} + +// Request message for `CreateReadSession`. +message CreateReadSessionRequest { + // Required. The request project that owns the session, in the form of + // `projects/{project_id}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; + + // Required. Session to be created. + ReadSession read_session = 2 [(google.api.field_behavior) = REQUIRED]; + + // Max initial number of streams. If unset or zero, the server will + // provide a value of streams so as to produce reasonable throughput. Must be + // non-negative. The number of streams may be lower than the requested number, + // depending on the amount parallelism that is reasonable for the table. Error + // will be returned if the max count is greater than the current system + // max limit of 1,000. + // + // Streams must be read starting from offset 0. + int32 max_stream_count = 3; +} + +// Request message for `ReadRows`. +message ReadRowsRequest { + // Required. Stream to read rows from. + string read_stream = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquerystorage.googleapis.com/ReadStream" + } + ]; + + // The offset requested must be less than the last row read from Read. + // Requesting a larger offset is undefined. If not specified, start reading + // from offset zero. + int64 offset = 2; +} + +// Information on if the current connection is being throttled. +message ThrottleState { + // How much this connection is being throttled. Zero means no throttling, + // 100 means fully throttled. + int32 throttle_percent = 1; +} + +// Estimated stream statistics for a given Stream. +message StreamStats { + message Progress { + // The fraction of rows assigned to the stream that have been processed by + // the server so far, not including the rows in the current response + // message. + // + // This value, along with `at_response_end`, can be used to interpolate + // the progress made as the rows in the message are being processed using + // the following formula: `at_response_start + (at_response_end - + // at_response_start) * rows_processed_from_response / rows_in_response`. + // + // Note that if a filter is provided, the `at_response_end` value of the + // previous response may not necessarily be equal to the + // `at_response_start` value of the current response. + double at_response_start = 1; + + // Similar to `at_response_start`, except that this value includes the + // rows in the current response. + double at_response_end = 2; + } + + // Represents the progress of the current stream. + Progress progress = 2; +} + +// Response from calling `ReadRows` may include row data, progress and +// throttling information. +message ReadRowsResponse { + // Row data is returned in format specified during session creation. + oneof rows { + // Serialized row data in AVRO format. + AvroRows avro_rows = 3; + + // Serialized row data in Arrow RecordBatch format. + ArrowRecordBatch arrow_record_batch = 4; + } + + // Number of serialized rows in the rows block. + int64 row_count = 6; + + // Statistics for the stream. + StreamStats stats = 2; + + // Throttling state. If unset, the latest response still describes + // the current throttling status. + ThrottleState throttle_state = 5; +} + +// Request message for `SplitReadStream`. +message SplitReadStreamRequest { + // Required. Name of the stream to split. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquerystorage.googleapis.com/ReadStream" + } + ]; + + // A value in the range (0.0, 1.0) that specifies the fractional point at + // which the original stream should be split. The actual split point is + // evaluated on pre-filtered rows, so if a filter is provided, then there is + // no guarantee that the division of the rows between the new child streams + // will be proportional to this fractional value. Additionally, because the + // server-side unit for assigning data is collections of rows, this fraction + // will always map to a data storage boundary on the server side. + double fraction = 2; +} + +// Response message for `SplitReadStream`. +message SplitReadStreamResponse { + // Primary stream, which contains the beginning portion of + // |original_stream|. An empty value indicates that the original stream can no + // longer be split. + ReadStream primary_stream = 1; + + // Remainder stream, which contains the tail of |original_stream|. An empty + // value indicates that the original stream can no longer be split. + ReadStream remainder_stream = 2; +} diff --git a/google/cloud/bigquery_storage_v1/proto/storage_pb2.py b/google/cloud/bigquery_storage_v1/proto/storage_pb2.py new file mode 100644 index 00000000..3821f077 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/storage_pb2.py @@ -0,0 +1,867 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: google/cloud/bigquery/storage_v1/proto/storage.proto + +import sys + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from google.api import client_pb2 as google_dot_api_dot_client__pb2 +from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 +from google.api import resource_pb2 as google_dot_api_dot_resource__pb2 +from google.cloud.bigquery_storage_v1.proto import ( + arrow_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_arrow__pb2, +) +from google.cloud.bigquery_storage_v1.proto import ( + avro_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_avro__pb2, +) +from google.cloud.bigquery_storage_v1.proto import ( + stream_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_stream__pb2, +) + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="google/cloud/bigquery/storage_v1/proto/storage.proto", + package="google.cloud.bigquery.storage.v1", + syntax="proto3", + serialized_options=_b( + "\n$com.google.cloud.bigquery.storage.v1B\014StorageProtoP\001ZGgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage\252\002 Google.Cloud.BigQuery.Storage.V1\312\002 Google\\Cloud\\BigQuery\\Storage\\V1\352AU\n\035bigquery.googleapis.com/Table\0224projects/{project}/datasets/{dataset}/tables/{table}" + ), + serialized_pb=_b( + '\n4google/cloud/bigquery/storage_v1/proto/storage.proto\x12 google.cloud.bigquery.storage.v1\x1a\x1cgoogle/api/annotations.proto\x1a\x17google/api/client.proto\x1a\x1fgoogle/api/field_behavior.proto\x1a\x19google/api/resource.proto\x1a\x32google/cloud/bigquery/storage_v1/proto/arrow.proto\x1a\x31google/cloud/bigquery/storage_v1/proto/avro.proto\x1a\x33google/cloud/bigquery/storage_v1/proto/stream.proto"\xc3\x01\n\x18\x43reateReadSessionRequest\x12\x43\n\x06parent\x18\x01 \x01(\tB3\xe0\x41\x02\xfa\x41-\n+cloudresourcemanager.googleapis.com/Project\x12H\n\x0cread_session\x18\x02 \x01(\x0b\x32-.google.cloud.bigquery.storage.v1.ReadSessionB\x03\xe0\x41\x02\x12\x18\n\x10max_stream_count\x18\x03 \x01(\x05"i\n\x0fReadRowsRequest\x12\x46\n\x0bread_stream\x18\x01 \x01(\tB1\xe0\x41\x02\xfa\x41+\n)bigquerystorage.googleapis.com/ReadStream\x12\x0e\n\x06offset\x18\x02 \x01(\x03")\n\rThrottleState\x12\x18\n\x10throttle_percent\x18\x01 \x01(\x05"\x97\x01\n\x0bStreamStats\x12H\n\x08progress\x18\x02 \x01(\x0b\x32\x36.google.cloud.bigquery.storage.v1.StreamStats.Progress\x1a>\n\x08Progress\x12\x19\n\x11\x61t_response_start\x18\x01 \x01(\x01\x12\x17\n\x0f\x61t_response_end\x18\x02 \x01(\x01"\xc7\x02\n\x10ReadRowsResponse\x12?\n\tavro_rows\x18\x03 \x01(\x0b\x32*.google.cloud.bigquery.storage.v1.AvroRowsH\x00\x12P\n\x12\x61rrow_record_batch\x18\x04 \x01(\x0b\x32\x32.google.cloud.bigquery.storage.v1.ArrowRecordBatchH\x00\x12\x11\n\trow_count\x18\x06 \x01(\x03\x12<\n\x05stats\x18\x02 \x01(\x0b\x32-.google.cloud.bigquery.storage.v1.StreamStats\x12G\n\x0ethrottle_state\x18\x05 \x01(\x0b\x32/.google.cloud.bigquery.storage.v1.ThrottleStateB\x06\n\x04rows"k\n\x16SplitReadStreamRequest\x12?\n\x04name\x18\x01 \x01(\tB1\xe0\x41\x02\xfa\x41+\n)bigquerystorage.googleapis.com/ReadStream\x12\x10\n\x08\x66raction\x18\x02 \x01(\x01"\xa7\x01\n\x17SplitReadStreamResponse\x12\x44\n\x0eprimary_stream\x18\x01 \x01(\x0b\x32,.google.cloud.bigquery.storage.v1.ReadStream\x12\x46\n\x10remainder_stream\x18\x02 \x01(\x0b\x32,.google.cloud.bigquery.storage.v1.ReadStream2\xc6\x06\n\x0c\x42igQueryRead\x12\xe9\x01\n\x11\x43reateReadSession\x12:.google.cloud.bigquery.storage.v1.CreateReadSessionRequest\x1a-.google.cloud.bigquery.storage.v1.ReadSession"i\x82\xd3\xe4\x93\x02<"7/v1/{read_session.table=projects/*/datasets/*/tables/*}:\x01*\xda\x41$parent,read_session,max_stream_count\x12\xcf\x01\n\x08ReadRows\x12\x31.google.cloud.bigquery.storage.v1.ReadRowsRequest\x1a\x32.google.cloud.bigquery.storage.v1.ReadRowsResponse"Z\x82\xd3\xe4\x93\x02?\x12=/v1/{read_stream=projects/*/locations/*/sessions/*/streams/*}\xda\x41\x12read_stream,offset0\x01\x12\xc6\x01\n\x0fSplitReadStream\x12\x38.google.cloud.bigquery.storage.v1.SplitReadStreamRequest\x1a\x39.google.cloud.bigquery.storage.v1.SplitReadStreamResponse">\x82\xd3\xe4\x93\x02\x38\x12\x36/v1/{name=projects/*/locations/*/sessions/*/streams/*}\x1a\xae\x01\xca\x41\x1e\x62igquerystorage.googleapis.com\xd2\x41\x89\x01https://www.googleapis.com/auth/bigquery,https://www.googleapis.com/auth/bigquery.readonly,https://www.googleapis.com/auth/cloud-platformB\x9d\x02\n$com.google.cloud.bigquery.storage.v1B\x0cStorageProtoP\x01ZGgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage\xaa\x02 Google.Cloud.BigQuery.Storage.V1\xca\x02 Google\\Cloud\\BigQuery\\Storage\\V1\xea\x41U\n\x1d\x62igquery.googleapis.com/Table\x12\x34projects/{project}/datasets/{dataset}/tables/{table}b\x06proto3' + ), + dependencies=[ + google_dot_api_dot_annotations__pb2.DESCRIPTOR, + google_dot_api_dot_client__pb2.DESCRIPTOR, + google_dot_api_dot_field__behavior__pb2.DESCRIPTOR, + google_dot_api_dot_resource__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_arrow__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_avro__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_stream__pb2.DESCRIPTOR, + ], +) + + +_CREATEREADSESSIONREQUEST = _descriptor.Descriptor( + name="CreateReadSessionRequest", + full_name="google.cloud.bigquery.storage.v1.CreateReadSessionRequest", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="parent", + full_name="google.cloud.bigquery.storage.v1.CreateReadSessionRequest.parent", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b( + "\340A\002\372A-\n+cloudresourcemanager.googleapis.com/Project" + ), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="read_session", + full_name="google.cloud.bigquery.storage.v1.CreateReadSessionRequest.read_session", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\002"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="max_stream_count", + full_name="google.cloud.bigquery.storage.v1.CreateReadSessionRequest.max_stream_count", + index=2, + number=3, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=362, + serialized_end=557, +) + + +_READROWSREQUEST = _descriptor.Descriptor( + name="ReadRowsRequest", + full_name="google.cloud.bigquery.storage.v1.ReadRowsRequest", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="read_stream", + full_name="google.cloud.bigquery.storage.v1.ReadRowsRequest.read_stream", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b( + "\340A\002\372A+\n)bigquerystorage.googleapis.com/ReadStream" + ), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="offset", + full_name="google.cloud.bigquery.storage.v1.ReadRowsRequest.offset", + index=1, + number=2, + type=3, + cpp_type=2, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=559, + serialized_end=664, +) + + +_THROTTLESTATE = _descriptor.Descriptor( + name="ThrottleState", + full_name="google.cloud.bigquery.storage.v1.ThrottleState", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="throttle_percent", + full_name="google.cloud.bigquery.storage.v1.ThrottleState.throttle_percent", + index=0, + number=1, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=666, + serialized_end=707, +) + + +_STREAMSTATS_PROGRESS = _descriptor.Descriptor( + name="Progress", + full_name="google.cloud.bigquery.storage.v1.StreamStats.Progress", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="at_response_start", + full_name="google.cloud.bigquery.storage.v1.StreamStats.Progress.at_response_start", + index=0, + number=1, + type=1, + cpp_type=5, + label=1, + has_default_value=False, + default_value=float(0), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="at_response_end", + full_name="google.cloud.bigquery.storage.v1.StreamStats.Progress.at_response_end", + index=1, + number=2, + type=1, + cpp_type=5, + label=1, + has_default_value=False, + default_value=float(0), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=799, + serialized_end=861, +) + +_STREAMSTATS = _descriptor.Descriptor( + name="StreamStats", + full_name="google.cloud.bigquery.storage.v1.StreamStats", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="progress", + full_name="google.cloud.bigquery.storage.v1.StreamStats.progress", + index=0, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[_STREAMSTATS_PROGRESS], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=710, + serialized_end=861, +) + + +_READROWSRESPONSE = _descriptor.Descriptor( + name="ReadRowsResponse", + full_name="google.cloud.bigquery.storage.v1.ReadRowsResponse", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="avro_rows", + full_name="google.cloud.bigquery.storage.v1.ReadRowsResponse.avro_rows", + index=0, + number=3, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="arrow_record_batch", + full_name="google.cloud.bigquery.storage.v1.ReadRowsResponse.arrow_record_batch", + index=1, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="row_count", + full_name="google.cloud.bigquery.storage.v1.ReadRowsResponse.row_count", + index=2, + number=6, + type=3, + cpp_type=2, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="stats", + full_name="google.cloud.bigquery.storage.v1.ReadRowsResponse.stats", + index=3, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="throttle_state", + full_name="google.cloud.bigquery.storage.v1.ReadRowsResponse.throttle_state", + index=4, + number=5, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name="rows", + full_name="google.cloud.bigquery.storage.v1.ReadRowsResponse.rows", + index=0, + containing_type=None, + fields=[], + ) + ], + serialized_start=864, + serialized_end=1191, +) + + +_SPLITREADSTREAMREQUEST = _descriptor.Descriptor( + name="SplitReadStreamRequest", + full_name="google.cloud.bigquery.storage.v1.SplitReadStreamRequest", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="name", + full_name="google.cloud.bigquery.storage.v1.SplitReadStreamRequest.name", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b( + "\340A\002\372A+\n)bigquerystorage.googleapis.com/ReadStream" + ), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="fraction", + full_name="google.cloud.bigquery.storage.v1.SplitReadStreamRequest.fraction", + index=1, + number=2, + type=1, + cpp_type=5, + label=1, + has_default_value=False, + default_value=float(0), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=1193, + serialized_end=1300, +) + + +_SPLITREADSTREAMRESPONSE = _descriptor.Descriptor( + name="SplitReadStreamResponse", + full_name="google.cloud.bigquery.storage.v1.SplitReadStreamResponse", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="primary_stream", + full_name="google.cloud.bigquery.storage.v1.SplitReadStreamResponse.primary_stream", + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="remainder_stream", + full_name="google.cloud.bigquery.storage.v1.SplitReadStreamResponse.remainder_stream", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=1303, + serialized_end=1470, +) + +_CREATEREADSESSIONREQUEST.fields_by_name[ + "read_session" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_stream__pb2._READSESSION +) +_STREAMSTATS_PROGRESS.containing_type = _STREAMSTATS +_STREAMSTATS.fields_by_name["progress"].message_type = _STREAMSTATS_PROGRESS +_READROWSRESPONSE.fields_by_name[ + "avro_rows" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_avro__pb2._AVROROWS +) +_READROWSRESPONSE.fields_by_name[ + "arrow_record_batch" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_arrow__pb2._ARROWRECORDBATCH +) +_READROWSRESPONSE.fields_by_name["stats"].message_type = _STREAMSTATS +_READROWSRESPONSE.fields_by_name["throttle_state"].message_type = _THROTTLESTATE +_READROWSRESPONSE.oneofs_by_name["rows"].fields.append( + _READROWSRESPONSE.fields_by_name["avro_rows"] +) +_READROWSRESPONSE.fields_by_name[ + "avro_rows" +].containing_oneof = _READROWSRESPONSE.oneofs_by_name["rows"] +_READROWSRESPONSE.oneofs_by_name["rows"].fields.append( + _READROWSRESPONSE.fields_by_name["arrow_record_batch"] +) +_READROWSRESPONSE.fields_by_name[ + "arrow_record_batch" +].containing_oneof = _READROWSRESPONSE.oneofs_by_name["rows"] +_SPLITREADSTREAMRESPONSE.fields_by_name[ + "primary_stream" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_stream__pb2._READSTREAM +) +_SPLITREADSTREAMRESPONSE.fields_by_name[ + "remainder_stream" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_stream__pb2._READSTREAM +) +DESCRIPTOR.message_types_by_name["CreateReadSessionRequest"] = _CREATEREADSESSIONREQUEST +DESCRIPTOR.message_types_by_name["ReadRowsRequest"] = _READROWSREQUEST +DESCRIPTOR.message_types_by_name["ThrottleState"] = _THROTTLESTATE +DESCRIPTOR.message_types_by_name["StreamStats"] = _STREAMSTATS +DESCRIPTOR.message_types_by_name["ReadRowsResponse"] = _READROWSRESPONSE +DESCRIPTOR.message_types_by_name["SplitReadStreamRequest"] = _SPLITREADSTREAMREQUEST +DESCRIPTOR.message_types_by_name["SplitReadStreamResponse"] = _SPLITREADSTREAMRESPONSE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +CreateReadSessionRequest = _reflection.GeneratedProtocolMessageType( + "CreateReadSessionRequest", + (_message.Message,), + dict( + DESCRIPTOR=_CREATEREADSESSIONREQUEST, + __module__="google.cloud.bigquery.storage_v1.proto.storage_pb2", + __doc__="""Request message for ``CreateReadSession``. + + + Attributes: + parent: + Required. The request project that owns the session, in the + form of ``projects/{project_id}``. + read_session: + Required. Session to be created. + max_stream_count: + Max initial number of streams. If unset or zero, the server + will provide a value of streams so as to produce reasonable + throughput. Must be non-negative. The number of streams may be + lower than the requested number, depending on the amount + parallelism that is reasonable for the table. Error will be + returned if the max count is greater than the current system + max limit of 1,000. Streams must be read starting from offset + 0. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.CreateReadSessionRequest) + ), +) +_sym_db.RegisterMessage(CreateReadSessionRequest) + +ReadRowsRequest = _reflection.GeneratedProtocolMessageType( + "ReadRowsRequest", + (_message.Message,), + dict( + DESCRIPTOR=_READROWSREQUEST, + __module__="google.cloud.bigquery.storage_v1.proto.storage_pb2", + __doc__="""Request message for ``ReadRows``. + + + Attributes: + read_stream: + Required. Stream to read rows from. + offset: + The offset requested must be less than the last row read from + Read. Requesting a larger offset is undefined. If not + specified, start reading from offset zero. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.ReadRowsRequest) + ), +) +_sym_db.RegisterMessage(ReadRowsRequest) + +ThrottleState = _reflection.GeneratedProtocolMessageType( + "ThrottleState", + (_message.Message,), + dict( + DESCRIPTOR=_THROTTLESTATE, + __module__="google.cloud.bigquery.storage_v1.proto.storage_pb2", + __doc__="""Information on if the current connection is being + throttled. + + + Attributes: + throttle_percent: + How much this connection is being throttled. Zero means no + throttling, 100 means fully throttled. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.ThrottleState) + ), +) +_sym_db.RegisterMessage(ThrottleState) + +StreamStats = _reflection.GeneratedProtocolMessageType( + "StreamStats", + (_message.Message,), + dict( + Progress=_reflection.GeneratedProtocolMessageType( + "Progress", + (_message.Message,), + dict( + DESCRIPTOR=_STREAMSTATS_PROGRESS, + __module__="google.cloud.bigquery.storage_v1.proto.storage_pb2", + __doc__="""Protocol buffer. + + Attributes: + at_response_start: + The fraction of rows assigned to the stream that have been + processed by the server so far, not including the rows in the + current response message. This value, along with + ``at_response_end``, can be used to interpolate the progress + made as the rows in the message are being processed using the + following formula: ``at_response_start + (at_response_end - + at_response_start) * rows_processed_from_response / + rows_in_response``. Note that if a filter is provided, the + ``at_response_end`` value of the previous response may not + necessarily be equal to the ``at_response_start`` value of the + current response. + at_response_end: + Similar to ``at_response_start``, except that this value + includes the rows in the current response. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.StreamStats.Progress) + ), + ), + DESCRIPTOR=_STREAMSTATS, + __module__="google.cloud.bigquery.storage_v1.proto.storage_pb2", + __doc__="""Estimated stream statistics for a given Stream. + + + Attributes: + progress: + Represents the progress of the current stream. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.StreamStats) + ), +) +_sym_db.RegisterMessage(StreamStats) +_sym_db.RegisterMessage(StreamStats.Progress) + +ReadRowsResponse = _reflection.GeneratedProtocolMessageType( + "ReadRowsResponse", + (_message.Message,), + dict( + DESCRIPTOR=_READROWSRESPONSE, + __module__="google.cloud.bigquery.storage_v1.proto.storage_pb2", + __doc__="""Response from calling ``ReadRows`` may include row data, + progress and throttling information. + + + Attributes: + rows: + Row data is returned in format specified during session + creation. + avro_rows: + Serialized row data in AVRO format. + arrow_record_batch: + Serialized row data in Arrow RecordBatch format. + row_count: + Number of serialized rows in the rows block. + stats: + Statistics for the stream. + throttle_state: + Throttling state. If unset, the latest response still + describes the current throttling status. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.ReadRowsResponse) + ), +) +_sym_db.RegisterMessage(ReadRowsResponse) + +SplitReadStreamRequest = _reflection.GeneratedProtocolMessageType( + "SplitReadStreamRequest", + (_message.Message,), + dict( + DESCRIPTOR=_SPLITREADSTREAMREQUEST, + __module__="google.cloud.bigquery.storage_v1.proto.storage_pb2", + __doc__="""Request message for ``SplitReadStream``. + + + Attributes: + name: + Required. Name of the stream to split. + fraction: + A value in the range (0.0, 1.0) that specifies the fractional + point at which the original stream should be split. The actual + split point is evaluated on pre-filtered rows, so if a filter + is provided, then there is no guarantee that the division of + the rows between the new child streams will be proportional to + this fractional value. Additionally, because the server-side + unit for assigning data is collections of rows, this fraction + will always map to a data storage boundary on the server side. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.SplitReadStreamRequest) + ), +) +_sym_db.RegisterMessage(SplitReadStreamRequest) + +SplitReadStreamResponse = _reflection.GeneratedProtocolMessageType( + "SplitReadStreamResponse", + (_message.Message,), + dict( + DESCRIPTOR=_SPLITREADSTREAMRESPONSE, + __module__="google.cloud.bigquery.storage_v1.proto.storage_pb2", + __doc__="""Response message for ``SplitReadStream``. + + + Attributes: + primary_stream: + Primary stream, which contains the beginning portion of + \|original\_stream\|. An empty value indicates that the + original stream can no longer be split. + remainder_stream: + Remainder stream, which contains the tail of + \|original\_stream\|. An empty value indicates that the + original stream can no longer be split. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.SplitReadStreamResponse) + ), +) +_sym_db.RegisterMessage(SplitReadStreamResponse) + + +DESCRIPTOR._options = None +_CREATEREADSESSIONREQUEST.fields_by_name["parent"]._options = None +_CREATEREADSESSIONREQUEST.fields_by_name["read_session"]._options = None +_READROWSREQUEST.fields_by_name["read_stream"]._options = None +_SPLITREADSTREAMREQUEST.fields_by_name["name"]._options = None + +_BIGQUERYREAD = _descriptor.ServiceDescriptor( + name="BigQueryRead", + full_name="google.cloud.bigquery.storage.v1.BigQueryRead", + file=DESCRIPTOR, + index=0, + serialized_options=_b( + "\312A\036bigquerystorage.googleapis.com\322A\211\001https://www.googleapis.com/auth/bigquery,https://www.googleapis.com/auth/bigquery.readonly,https://www.googleapis.com/auth/cloud-platform" + ), + serialized_start=1473, + serialized_end=2311, + methods=[ + _descriptor.MethodDescriptor( + name="CreateReadSession", + full_name="google.cloud.bigquery.storage.v1.BigQueryRead.CreateReadSession", + index=0, + containing_service=None, + input_type=_CREATEREADSESSIONREQUEST, + output_type=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_stream__pb2._READSESSION, + serialized_options=_b( + '\202\323\344\223\002<"7/v1/{read_session.table=projects/*/datasets/*/tables/*}:\001*\332A$parent,read_session,max_stream_count' + ), + ), + _descriptor.MethodDescriptor( + name="ReadRows", + full_name="google.cloud.bigquery.storage.v1.BigQueryRead.ReadRows", + index=1, + containing_service=None, + input_type=_READROWSREQUEST, + output_type=_READROWSRESPONSE, + serialized_options=_b( + "\202\323\344\223\002?\022=/v1/{read_stream=projects/*/locations/*/sessions/*/streams/*}\332A\022read_stream,offset" + ), + ), + _descriptor.MethodDescriptor( + name="SplitReadStream", + full_name="google.cloud.bigquery.storage.v1.BigQueryRead.SplitReadStream", + index=2, + containing_service=None, + input_type=_SPLITREADSTREAMREQUEST, + output_type=_SPLITREADSTREAMRESPONSE, + serialized_options=_b( + "\202\323\344\223\0028\0226/v1/{name=projects/*/locations/*/sessions/*/streams/*}" + ), + ), + ], +) +_sym_db.RegisterServiceDescriptor(_BIGQUERYREAD) + +DESCRIPTOR.services_by_name["BigQueryRead"] = _BIGQUERYREAD + +# @@protoc_insertion_point(module_scope) diff --git a/google/cloud/bigquery_storage_v1/proto/storage_pb2_grpc.py b/google/cloud/bigquery_storage_v1/proto/storage_pb2_grpc.py new file mode 100644 index 00000000..271befe6 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/storage_pb2_grpc.py @@ -0,0 +1,125 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +from google.cloud.bigquery_storage_v1.proto import ( + storage_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2, +) +from google.cloud.bigquery_storage_v1.proto import ( + stream_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_stream__pb2, +) + + +class BigQueryReadStub(object): + """BigQuery Read API. + + The Read API can be used to read data from BigQuery. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.CreateReadSession = channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryRead/CreateReadSession", + request_serializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.CreateReadSessionRequest.SerializeToString, + response_deserializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_stream__pb2.ReadSession.FromString, + ) + self.ReadRows = channel.unary_stream( + "/google.cloud.bigquery.storage.v1.BigQueryRead/ReadRows", + request_serializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.ReadRowsRequest.SerializeToString, + response_deserializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.ReadRowsResponse.FromString, + ) + self.SplitReadStream = channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryRead/SplitReadStream", + request_serializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.SplitReadStreamRequest.SerializeToString, + response_deserializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.SplitReadStreamResponse.FromString, + ) + + +class BigQueryReadServicer(object): + """BigQuery Read API. + + The Read API can be used to read data from BigQuery. + """ + + def CreateReadSession(self, request, context): + """Creates a new read session. A read session divides the contents of a + BigQuery table into one or more streams, which can then be used to read + data from the table. The read session also specifies properties of the + data to be read, such as a list of columns or a push-down filter describing + the rows to be returned. + + A particular row can be read by at most one stream. When the caller has + reached the end of each stream in the session, then all the data in the + table has been read. + + Data is assigned to each stream such that roughly the same number of + rows can be read from each stream. Because the server-side unit for + assigning data is collections of rows, the API does not guarantee that + each stream will return the same number or rows. Additionally, the + limits are enforced based on the number of pre-filtered rows, so some + filters can lead to lopsided assignments. + + Read sessions automatically expire 24 hours after they are created and do + not require manual clean-up by the caller. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def ReadRows(self, request, context): + """Reads rows from the stream in the format prescribed by the ReadSession. + Each response contains one or more table rows, up to a maximum of 100 MiB + per response; read requests which attempt to read individual rows larger + than 100 MiB will fail. + + Each request also returns a set of stream statistics reflecting the current + state of the stream. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def SplitReadStream(self, request, context): + """Splits a given `ReadStream` into two `ReadStream` objects. These + `ReadStream` objects are referred to as the primary and the residual + streams of the split. The original `ReadStream` can still be read from in + the same manner as before. Both of the returned `ReadStream` objects can + also be read from, and the rows returned by both child streams will be + the same as the rows read from the original stream. + + Moreover, the two child streams will be allocated back-to-back in the + original `ReadStream`. Concretely, it is guaranteed that for streams + original, primary, and residual, that original[0-j] = primary[0-j] and + original[j-n] = residual[0-m] once the streams have been read to + completion. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_BigQueryReadServicer_to_server(servicer, server): + rpc_method_handlers = { + "CreateReadSession": grpc.unary_unary_rpc_method_handler( + servicer.CreateReadSession, + request_deserializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.CreateReadSessionRequest.FromString, + response_serializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_stream__pb2.ReadSession.SerializeToString, + ), + "ReadRows": grpc.unary_stream_rpc_method_handler( + servicer.ReadRows, + request_deserializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.ReadRowsRequest.FromString, + response_serializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.ReadRowsResponse.SerializeToString, + ), + "SplitReadStream": grpc.unary_unary_rpc_method_handler( + servicer.SplitReadStream, + request_deserializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.SplitReadStreamRequest.FromString, + response_serializer=google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_storage__pb2.SplitReadStreamResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "google.cloud.bigquery.storage.v1.BigQueryRead", rpc_method_handlers + ) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/google/cloud/bigquery_storage_v1/proto/stream.proto b/google/cloud/bigquery_storage_v1/proto/stream.proto new file mode 100644 index 00000000..3ef32c13 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/stream.proto @@ -0,0 +1,138 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.cloud.bigquery.storage.v1; + +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/cloud/bigquery/storage/v1/arrow.proto"; +import "google/cloud/bigquery/storage/v1/avro.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.BigQuery.Storage.V1"; +option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage"; +option java_multiple_files = true; +option java_outer_classname = "StreamProto"; +option java_package = "com.google.cloud.bigquery.storage.v1"; +option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1"; + +// Data format for input or output data. +enum DataFormat { + DATA_FORMAT_UNSPECIFIED = 0; + + // Avro is a standard open source row based file format. + // See https://avro.apache.org/ for more details. + AVRO = 1; + + // Arrow is a standard open source column-based message format. + // See https://arrow.apache.org/ for more details. + ARROW = 2; +} + +// Information about the ReadSession. +message ReadSession { + option (google.api.resource) = { + type: "bigquerystorage.googleapis.com/ReadSession" + pattern: "projects/{project}/locations/{location}/sessions/{session}" + }; + + // Additional attributes when reading a table. + message TableModifiers { + // The snapshot time of the table. If not set, interpreted as now. + google.protobuf.Timestamp snapshot_time = 1; + } + + // Options dictating how we read a table. + message TableReadOptions { + // Names of the fields in the table that should be read. If empty, all + // fields will be read. If the specified field is a nested field, all + // the sub-fields in the field will be selected. The output field order is + // unrelated to the order of fields in selected_fields. + repeated string selected_fields = 1; + + // SQL text filtering statement, similar to a WHERE clause in a query. + // Currently, only a single predicate that is a comparison between a column + // and a constant value is supported. Aggregates are not supported. + // + // Examples: "int_field > 5" + // "date_field = CAST('2014-9-27' as DATE)" + // "nullable_field is not NULL" + // "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))" + // "numeric_field BETWEEN 1.0 AND 5.0" + string row_restriction = 2; + } + + // Output only. Unique identifier for the session, in the form + // `projects/{project_id}/locations/{location}/sessions/{session_id}`. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Time at which the session becomes invalid. After this time, subsequent + // requests to read this Session will return errors. The expire_time is + // automatically assigned and currently cannot be specified or updated. + google.protobuf.Timestamp expire_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Immutable. Data format of the output data. + DataFormat data_format = 3 [(google.api.field_behavior) = IMMUTABLE]; + + // The schema for the read. If read_options.selected_fields is set, the + // schema may be different from the table schema as it will only contain + // the selected fields. + oneof schema { + // Output only. Avro schema. + AvroSchema avro_schema = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Arrow schema. + ArrowSchema arrow_schema = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + } + + // Immutable. Table that this ReadSession is reading from, in the form + // `projects/{project_id}/datasets/{dataset_id}/tables/{table_id} + string table = 6 [ + (google.api.field_behavior) = IMMUTABLE, + (google.api.resource_reference) = { + type: "bigquery.googleapis.com/Table" + } + ]; + + // Optional. Any modifiers which are applied when reading from the specified table. + TableModifiers table_modifiers = 7 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Read options for this session (e.g. column selection, filters). + TableReadOptions read_options = 8 [(google.api.field_behavior) = OPTIONAL]; + + // Output only. A list of streams created with the session. + // + // At least one stream is created with the session. In the future, larger + // request_stream_count values *may* result in this list being unpopulated, + // in that case, the user will need to use a List method to get the streams + // instead, which is not yet available. + repeated ReadStream streams = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Information about a single stream that gets data out of the storage system. +// Most of the information about `ReadStream` instances is aggregated, making +// `ReadStream` lightweight. +message ReadStream { + option (google.api.resource) = { + type: "bigquerystorage.googleapis.com/ReadStream" + pattern: "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}" + }; + + // Output only. Name of the stream, in the form + // `projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}`. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; +} diff --git a/google/cloud/bigquery_storage_v1/proto/stream_pb2.py b/google/cloud/bigquery_storage_v1/proto/stream_pb2.py new file mode 100644 index 00000000..46f1177b --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/stream_pb2.py @@ -0,0 +1,587 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: google/cloud/bigquery/storage_v1/proto/stream.proto + +import sys + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 +from google.api import resource_pb2 as google_dot_api_dot_resource__pb2 +from google.cloud.bigquery.storage_v1.proto import ( + arrow_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_arrow__pb2, +) +from google.cloud.bigquery.storage_v1.proto import ( + avro_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_avro__pb2, +) +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="google/cloud/bigquery/storage_v1/proto/stream.proto", + package="google.cloud.bigquery.storage.v1", + syntax="proto3", + serialized_options=_b( + "\n$com.google.cloud.bigquery.storage.v1B\013StreamProtoP\001ZGgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage\252\002 Google.Cloud.BigQuery.Storage.V1\312\002 Google\\Cloud\\BigQuery\\Storage\\V1" + ), + serialized_pb=_b( + '\n3google/cloud/bigquery/storage_v1/proto/stream.proto\x12 google.cloud.bigquery.storage.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x19google/api/resource.proto\x1a\x32google/cloud/bigquery/storage_v1/proto/arrow.proto\x1a\x31google/cloud/bigquery/storage_v1/proto/avro.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\xe7\x06\n\x0bReadSession\x12\x11\n\x04name\x18\x01 \x01(\tB\x03\xe0\x41\x03\x12\x34\n\x0b\x65xpire_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03\x12\x46\n\x0b\x64\x61ta_format\x18\x03 \x01(\x0e\x32,.google.cloud.bigquery.storage.v1.DataFormatB\x03\xe0\x41\x05\x12H\n\x0b\x61vro_schema\x18\x04 \x01(\x0b\x32,.google.cloud.bigquery.storage.v1.AvroSchemaB\x03\xe0\x41\x03H\x00\x12J\n\x0c\x61rrow_schema\x18\x05 \x01(\x0b\x32-.google.cloud.bigquery.storage.v1.ArrowSchemaB\x03\xe0\x41\x03H\x00\x12\x34\n\x05table\x18\x06 \x01(\tB%\xe0\x41\x05\xfa\x41\x1f\n\x1d\x62igquery.googleapis.com/Table\x12Z\n\x0ftable_modifiers\x18\x07 \x01(\x0b\x32<.google.cloud.bigquery.storage.v1.ReadSession.TableModifiersB\x03\xe0\x41\x01\x12Y\n\x0cread_options\x18\x08 \x01(\x0b\x32>.google.cloud.bigquery.storage.v1.ReadSession.TableReadOptionsB\x03\xe0\x41\x01\x12\x42\n\x07streams\x18\n \x03(\x0b\x32,.google.cloud.bigquery.storage.v1.ReadStreamB\x03\xe0\x41\x03\x1a\x43\n\x0eTableModifiers\x12\x31\n\rsnapshot_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\x44\n\x10TableReadOptions\x12\x17\n\x0fselected_fields\x18\x01 \x03(\t\x12\x17\n\x0frow_restriction\x18\x02 \x01(\t:k\xea\x41h\n*bigquerystorage.googleapis.com/ReadSession\x12:projects/{project}/locations/{location}/sessions/{session}B\x08\n\x06schema"\x9c\x01\n\nReadStream\x12\x11\n\x04name\x18\x01 \x01(\tB\x03\xe0\x41\x03:{\xea\x41x\n)bigquerystorage.googleapis.com/ReadStream\x12Kprojects/{project}/locations/{location}/sessions/{session}/streams/{stream}*>\n\nDataFormat\x12\x1b\n\x17\x44\x41TA_FORMAT_UNSPECIFIED\x10\x00\x12\x08\n\x04\x41VRO\x10\x01\x12\t\n\x05\x41RROW\x10\x02\x42\xc4\x01\n$com.google.cloud.bigquery.storage.v1B\x0bStreamProtoP\x01ZGgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1;storage\xaa\x02 Google.Cloud.BigQuery.Storage.V1\xca\x02 Google\\Cloud\\BigQuery\\Storage\\V1b\x06proto3' + ), + dependencies=[ + google_dot_api_dot_field__behavior__pb2.DESCRIPTOR, + google_dot_api_dot_resource__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_arrow__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_avro__pb2.DESCRIPTOR, + google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR, + ], +) + +_DATAFORMAT = _descriptor.EnumDescriptor( + name="DataFormat", + full_name="google.cloud.bigquery.storage.v1.DataFormat", + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name="DATA_FORMAT_UNSPECIFIED", + index=0, + number=0, + serialized_options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name="AVRO", index=1, number=1, serialized_options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ARROW", index=2, number=2, serialized_options=None, type=None + ), + ], + containing_type=None, + serialized_options=None, + serialized_start=1318, + serialized_end=1380, +) +_sym_db.RegisterEnumDescriptor(_DATAFORMAT) + +DataFormat = enum_type_wrapper.EnumTypeWrapper(_DATAFORMAT) +DATA_FORMAT_UNSPECIFIED = 0 +AVRO = 1 +ARROW = 2 + + +_READSESSION_TABLEMODIFIERS = _descriptor.Descriptor( + name="TableModifiers", + full_name="google.cloud.bigquery.storage.v1.ReadSession.TableModifiers", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="snapshot_time", + full_name="google.cloud.bigquery.storage.v1.ReadSession.TableModifiers.snapshot_time", + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=901, + serialized_end=968, +) + +_READSESSION_TABLEREADOPTIONS = _descriptor.Descriptor( + name="TableReadOptions", + full_name="google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="selected_fields", + full_name="google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions.selected_fields", + index=0, + number=1, + type=9, + cpp_type=9, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="row_restriction", + full_name="google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions.row_restriction", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=970, + serialized_end=1038, +) + +_READSESSION = _descriptor.Descriptor( + name="ReadSession", + full_name="google.cloud.bigquery.storage.v1.ReadSession", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="name", + full_name="google.cloud.bigquery.storage.v1.ReadSession.name", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="expire_time", + full_name="google.cloud.bigquery.storage.v1.ReadSession.expire_time", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="data_format", + full_name="google.cloud.bigquery.storage.v1.ReadSession.data_format", + index=2, + number=3, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\005"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="avro_schema", + full_name="google.cloud.bigquery.storage.v1.ReadSession.avro_schema", + index=3, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="arrow_schema", + full_name="google.cloud.bigquery.storage.v1.ReadSession.arrow_schema", + index=4, + number=5, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="table", + full_name="google.cloud.bigquery.storage.v1.ReadSession.table", + index=5, + number=6, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b( + "\340A\005\372A\037\n\035bigquery.googleapis.com/Table" + ), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="table_modifiers", + full_name="google.cloud.bigquery.storage.v1.ReadSession.table_modifiers", + index=6, + number=7, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\001"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="read_options", + full_name="google.cloud.bigquery.storage.v1.ReadSession.read_options", + index=7, + number=8, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\001"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="streams", + full_name="google.cloud.bigquery.storage.v1.ReadSession.streams", + index=8, + number=10, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[_READSESSION_TABLEMODIFIERS, _READSESSION_TABLEREADOPTIONS], + enum_types=[], + serialized_options=_b( + "\352Ah\n*bigquerystorage.googleapis.com/ReadSession\022:projects/{project}/locations/{location}/sessions/{session}" + ), + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name="schema", + full_name="google.cloud.bigquery.storage.v1.ReadSession.schema", + index=0, + containing_type=None, + fields=[], + ) + ], + serialized_start=286, + serialized_end=1157, +) + + +_READSTREAM = _descriptor.Descriptor( + name="ReadStream", + full_name="google.cloud.bigquery.storage.v1.ReadStream", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="name", + full_name="google.cloud.bigquery.storage.v1.ReadStream.name", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=_b( + "\352Ax\n)bigquerystorage.googleapis.com/ReadStream\022Kprojects/{project}/locations/{location}/sessions/{session}/streams/{stream}" + ), + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=1160, + serialized_end=1316, +) + +_READSESSION_TABLEMODIFIERS.fields_by_name[ + "snapshot_time" +].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +_READSESSION_TABLEMODIFIERS.containing_type = _READSESSION +_READSESSION_TABLEREADOPTIONS.containing_type = _READSESSION +_READSESSION.fields_by_name[ + "expire_time" +].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +_READSESSION.fields_by_name["data_format"].enum_type = _DATAFORMAT +_READSESSION.fields_by_name[ + "avro_schema" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_avro__pb2._AVROSCHEMA +) +_READSESSION.fields_by_name[ + "arrow_schema" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_arrow__pb2._ARROWSCHEMA +) +_READSESSION.fields_by_name[ + "table_modifiers" +].message_type = _READSESSION_TABLEMODIFIERS +_READSESSION.fields_by_name["read_options"].message_type = _READSESSION_TABLEREADOPTIONS +_READSESSION.fields_by_name["streams"].message_type = _READSTREAM +_READSESSION.oneofs_by_name["schema"].fields.append( + _READSESSION.fields_by_name["avro_schema"] +) +_READSESSION.fields_by_name[ + "avro_schema" +].containing_oneof = _READSESSION.oneofs_by_name["schema"] +_READSESSION.oneofs_by_name["schema"].fields.append( + _READSESSION.fields_by_name["arrow_schema"] +) +_READSESSION.fields_by_name[ + "arrow_schema" +].containing_oneof = _READSESSION.oneofs_by_name["schema"] +DESCRIPTOR.message_types_by_name["ReadSession"] = _READSESSION +DESCRIPTOR.message_types_by_name["ReadStream"] = _READSTREAM +DESCRIPTOR.enum_types_by_name["DataFormat"] = _DATAFORMAT +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +ReadSession = _reflection.GeneratedProtocolMessageType( + "ReadSession", + (_message.Message,), + dict( + TableModifiers=_reflection.GeneratedProtocolMessageType( + "TableModifiers", + (_message.Message,), + dict( + DESCRIPTOR=_READSESSION_TABLEMODIFIERS, + __module__="google.cloud.bigquery.storage_v1.proto.stream_pb2", + __doc__="""Additional attributes when reading a table. + + + Attributes: + snapshot_time: + The snapshot time of the table. If not set, interpreted as + now. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.ReadSession.TableModifiers) + ), + ), + TableReadOptions=_reflection.GeneratedProtocolMessageType( + "TableReadOptions", + (_message.Message,), + dict( + DESCRIPTOR=_READSESSION_TABLEREADOPTIONS, + __module__="google.cloud.bigquery.storage_v1.proto.stream_pb2", + __doc__="""Options dictating how we read a table. + + + Attributes: + selected_fields: + Names of the fields in the table that should be read. If + empty, all fields will be read. If the specified field is a + nested field, all the sub-fields in the field will be + selected. The output field order is unrelated to the order of + fields in selected\_fields. + row_restriction: + SQL text filtering statement, similar to a WHERE clause in a + query. Currently, only a single predicate that is a comparison + between a column and a constant value is supported. Aggregates + are not supported. Examples: "int\_field > 5" "date\_field = + CAST('2014-9-27' as DATE)" "nullable\_field is not NULL" + "st\_equals(geo\_field, st\_geofromtext("POINT(2, 2)"))" + "numeric\_field BETWEEN 1.0 AND 5.0" + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions) + ), + ), + DESCRIPTOR=_READSESSION, + __module__="google.cloud.bigquery.storage_v1.proto.stream_pb2", + __doc__="""Information about the ReadSession. + + + Attributes: + name: + Output only. Unique identifier for the session, in the form `` + projects/{project_id}/locations/{location}/sessions/{session_i + d}``. + expire_time: + Output only. Time at which the session becomes invalid. After + this time, subsequent requests to read this Session will + return errors. The expire\_time is automatically assigned and + currently cannot be specified or updated. + data_format: + Immutable. Data format of the output data. + schema: + The schema for the read. If read\_options.selected\_fields is + set, the schema may be different from the table schema as it + will only contain the selected fields. + avro_schema: + Output only. Avro schema. + arrow_schema: + Output only. Arrow schema. + table: + Immutable. Table that this ReadSession is reading from, in the + form \`projects/{project\_id}/datasets/{dataset\_id}/tables/{t + able\_id} + table_modifiers: + Optional. Any modifiers which are applied when reading from + the specified table. + read_options: + Optional. Read options for this session (e.g. column + selection, filters). + streams: + Output only. A list of streams created with the session. At + least one stream is created with the session. In the future, + larger request\_stream\_count values *may* result in this list + being unpopulated, in that case, the user will need to use a + List method to get the streams instead, which is not yet + available. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.ReadSession) + ), +) +_sym_db.RegisterMessage(ReadSession) +_sym_db.RegisterMessage(ReadSession.TableModifiers) +_sym_db.RegisterMessage(ReadSession.TableReadOptions) + +ReadStream = _reflection.GeneratedProtocolMessageType( + "ReadStream", + (_message.Message,), + dict( + DESCRIPTOR=_READSTREAM, + __module__="google.cloud.bigquery.storage_v1.proto.stream_pb2", + __doc__="""Information about a single stream that gets data out of + the storage system. Most of the information about ``ReadStream`` + instances is aggregated, making ``ReadStream`` lightweight. + + + Attributes: + name: + Output only. Name of the stream, in the form ``projects/{proje + ct_id}/locations/{location}/sessions/{session_id}/streams/{str + eam_id}``. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1.ReadStream) + ), +) +_sym_db.RegisterMessage(ReadStream) + + +DESCRIPTOR._options = None +_READSESSION.fields_by_name["name"]._options = None +_READSESSION.fields_by_name["expire_time"]._options = None +_READSESSION.fields_by_name["data_format"]._options = None +_READSESSION.fields_by_name["avro_schema"]._options = None +_READSESSION.fields_by_name["arrow_schema"]._options = None +_READSESSION.fields_by_name["table"]._options = None +_READSESSION.fields_by_name["table_modifiers"]._options = None +_READSESSION.fields_by_name["read_options"]._options = None +_READSESSION.fields_by_name["streams"]._options = None +_READSESSION._options = None +_READSTREAM.fields_by_name["name"]._options = None +_READSTREAM._options = None +# @@protoc_insertion_point(module_scope) diff --git a/google/cloud/bigquery_storage_v1/proto/stream_pb2_grpc.py b/google/cloud/bigquery_storage_v1/proto/stream_pb2_grpc.py new file mode 100644 index 00000000..07cb78fe --- /dev/null +++ b/google/cloud/bigquery_storage_v1/proto/stream_pb2_grpc.py @@ -0,0 +1,2 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc diff --git a/google/cloud/bigquery_storage_v1/types.py b/google/cloud/bigquery_storage_v1/types.py new file mode 100644 index 00000000..881792e8 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/types.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import absolute_import +import sys + +from google.api_core.protobuf_helpers import get_messages + +from google.cloud.bigquery_storage_v1.proto import arrow_pb2 +from google.cloud.bigquery_storage_v1.proto import avro_pb2 +from google.cloud.bigquery_storage_v1.proto import storage_pb2 +from google.cloud.bigquery_storage_v1.proto import stream_pb2 +from google.protobuf import timestamp_pb2 + + +_shared_modules = [timestamp_pb2] + +_local_modules = [arrow_pb2, avro_pb2, storage_pb2, stream_pb2] + +names = [] + +for module in _shared_modules: # pragma: NO COVER + for name, message in get_messages(module).items(): + setattr(sys.modules[__name__], name, message) + names.append(name) +for module in _local_modules: + for name, message in get_messages(module).items(): + message.__module__ = "google.cloud.bigquery_storage_v1.types" + setattr(sys.modules[__name__], name, message) + names.append(name) + + +__all__ = tuple(sorted(names)) diff --git a/google/cloud/bigquery_storage_v1beta2/gapic/__init__.py b/google/cloud/bigquery_storage_v1beta2/gapic/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/google/cloud/bigquery_storage_v1beta2/gapic/big_query_read_client.py b/google/cloud/bigquery_storage_v1beta2/gapic/big_query_read_client.py new file mode 100644 index 00000000..4ac9408d --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/gapic/big_query_read_client.py @@ -0,0 +1,459 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Accesses the google.cloud.bigquery.storage.v1beta2 BigQueryRead API.""" + +import pkg_resources +import warnings + +from google.oauth2 import service_account +import google.api_core.client_options +import google.api_core.gapic_v1.client_info +import google.api_core.gapic_v1.config +import google.api_core.gapic_v1.method +import google.api_core.path_template +import google.api_core.gapic_v1.routing_header +import google.api_core.grpc_helpers +import grpc + +from google.cloud.bigquery_storage_v1beta2.gapic import big_query_read_client_config +from google.cloud.bigquery_storage_v1beta2.gapic import enums +from google.cloud.bigquery_storage_v1beta2.gapic.transports import ( + big_query_read_grpc_transport, +) +from google.cloud.bigquery_storage_v1beta2.proto import storage_pb2 +from google.cloud.bigquery_storage_v1beta2.proto import storage_pb2_grpc +from google.cloud.bigquery_storage_v1beta2.proto import stream_pb2 + + +_GAPIC_LIBRARY_VERSION = pkg_resources.get_distribution( + "google-cloud-bigquery-storage" +).version + + +class BigQueryReadClient(object): + """ + BigQuery Read API. + + The Read API can be used to read data from BigQuery. + """ + + SERVICE_ADDRESS = "bigquerystorage.googleapis.com:443" + """The default address of the service.""" + + # The name of the interface for this client. This is the key used to + # find the method configuration in the client_config dictionary. + _INTERFACE_NAME = "google.cloud.bigquery.storage.v1beta2.BigQueryRead" + + @classmethod + def from_service_account_file(cls, filename, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file(filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + def __init__( + self, + transport=None, + channel=None, + credentials=None, + client_config=None, + client_info=None, + client_options=None, + ): + """Constructor. + + Args: + transport (Union[~.BigQueryReadGrpcTransport, + Callable[[~.Credentials, type], ~.BigQueryReadGrpcTransport]): A transport + instance, responsible for actually making the API calls. + The default transport uses the gRPC protocol. + This argument may also be a callable which returns a + transport instance. Callables will be sent the credentials + as the first argument and the default transport class as + the second argument. + channel (grpc.Channel): DEPRECATED. A ``Channel`` instance + through which to make calls. This argument is mutually exclusive + with ``credentials``; providing both will raise an exception. + credentials (google.auth.credentials.Credentials): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is mutually exclusive with providing a + transport instance to ``transport``; doing so will raise + an exception. + client_config (dict): DEPRECATED. A dictionary of call options for + each method. If not specified, the default configuration is used. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + client_options (Union[dict, google.api_core.client_options.ClientOptions]): + Client options used to set user options on the client. API Endpoint + should be set through client_options. + """ + # Raise deprecation warnings for things we want to go away. + if client_config is not None: + warnings.warn( + "The `client_config` argument is deprecated.", + PendingDeprecationWarning, + stacklevel=2, + ) + else: + client_config = big_query_read_client_config.config + + if channel: + warnings.warn( + "The `channel` argument is deprecated; use " "`transport` instead.", + PendingDeprecationWarning, + stacklevel=2, + ) + + api_endpoint = self.SERVICE_ADDRESS + if client_options: + if type(client_options) == dict: + client_options = google.api_core.client_options.from_dict( + client_options + ) + if client_options.api_endpoint: + api_endpoint = client_options.api_endpoint + + # Instantiate the transport. + # The transport is responsible for handling serialization and + # deserialization and actually sending data to the service. + if transport: # pragma: no cover + if callable(transport): + self.transport = transport( + credentials=credentials, + default_class=big_query_read_grpc_transport.BigQueryReadGrpcTransport, + address=api_endpoint, + ) + else: + if credentials: + raise ValueError( + "Received both a transport instance and " + "credentials; these are mutually exclusive." + ) + self.transport = transport + else: + self.transport = big_query_read_grpc_transport.BigQueryReadGrpcTransport( + address=api_endpoint, channel=channel, credentials=credentials + ) + + if client_info is None: + client_info = google.api_core.gapic_v1.client_info.ClientInfo( + gapic_version=_GAPIC_LIBRARY_VERSION + ) + else: + client_info.gapic_version = _GAPIC_LIBRARY_VERSION + self._client_info = client_info + + # Parse out the default settings for retry and timeout for each RPC + # from the client configuration. + # (Ordinarily, these are the defaults specified in the `*_config.py` + # file next to this one.) + self._method_configs = google.api_core.gapic_v1.config.parse_method_configs( + client_config["interfaces"][self._INTERFACE_NAME] + ) + + # Save a dictionary of cached API call functions. + # These are the actual callables which invoke the proper + # transport methods, wrapped with `wrap_method` to add retry, + # timeout, and the like. + self._inner_api_calls = {} + + # Service calls + def create_read_session( + self, + parent=None, + read_session=None, + max_stream_count=None, + retry=google.api_core.gapic_v1.method.DEFAULT, + timeout=google.api_core.gapic_v1.method.DEFAULT, + metadata=None, + ): + """ + Creates a new read session. A read session divides the contents of a + BigQuery table into one or more streams, which can then be used to read + data from the table. The read session also specifies properties of the + data to be read, such as a list of columns or a push-down filter describing + the rows to be returned. + + A particular row can be read by at most one stream. When the caller has + reached the end of each stream in the session, then all the data in the + table has been read. + + Data is assigned to each stream such that roughly the same number of + rows can be read from each stream. Because the server-side unit for + assigning data is collections of rows, the API does not guarantee that + each stream will return the same number or rows. Additionally, the + limits are enforced based on the number of pre-filtered rows, so some + filters can lead to lopsided assignments. + + Read sessions automatically expire 24 hours after they are created and do + not require manual clean-up by the caller. + + Example: + >>> from google.cloud import bigquery_storage_v1beta2 + >>> + >>> client = bigquery_storage_v1beta2.BigQueryReadClient() + >>> + >>> response = client.create_read_session() + + Args: + parent (str): Required. The request project that owns the session, in the form of + ``projects/{project_id}``. + read_session (Union[dict, ~google.cloud.bigquery_storage_v1beta2.types.ReadSession]): Required. Session to be created. + + If a dict is provided, it must be of the same form as the protobuf + message :class:`~google.cloud.bigquery_storage_v1beta2.types.ReadSession` + max_stream_count (int): Max initial number of streams. If unset or zero, the server will + provide a value of streams so as to produce reasonable throughput. Must be + non-negative. The number of streams may be lower than the requested number, + depending on the amount parallelism that is reasonable for the table. Error + will be returned if the max count is greater than the current system + max limit of 1,000. + + Streams must be read starting from offset 0. + retry (Optional[google.api_core.retry.Retry]): A retry object used + to retry requests. If ``None`` is specified, requests will + be retried using a default configuration. + timeout (Optional[float]): The amount of time, in seconds, to wait + for the request to complete. Note that if ``retry`` is + specified, the timeout applies to each individual attempt. + metadata (Optional[Sequence[Tuple[str, str]]]): Additional metadata + that is provided to the method. + + Returns: + A :class:`~google.cloud.bigquery_storage_v1beta2.types.ReadSession` instance. + + Raises: + google.api_core.exceptions.GoogleAPICallError: If the request + failed for any reason. + google.api_core.exceptions.RetryError: If the request failed due + to a retryable error and retry attempts failed. + ValueError: If the parameters are invalid. + """ + # Wrap the transport method to add retry and timeout logic. + if "create_read_session" not in self._inner_api_calls: + self._inner_api_calls[ + "create_read_session" + ] = google.api_core.gapic_v1.method.wrap_method( + self.transport.create_read_session, + default_retry=self._method_configs["CreateReadSession"].retry, + default_timeout=self._method_configs["CreateReadSession"].timeout, + client_info=self._client_info, + ) + + request = storage_pb2.CreateReadSessionRequest( + parent=parent, read_session=read_session, max_stream_count=max_stream_count + ) + if metadata is None: + metadata = [] + metadata = list(metadata) + try: + routing_header = [("read_session.table", read_session.table)] + except AttributeError: + pass + else: + routing_metadata = google.api_core.gapic_v1.routing_header.to_grpc_metadata( + routing_header + ) + metadata.append(routing_metadata) # pragma: no cover + + return self._inner_api_calls["create_read_session"]( + request, retry=retry, timeout=timeout, metadata=metadata + ) + + def read_rows( + self, + read_stream=None, + offset=None, + retry=google.api_core.gapic_v1.method.DEFAULT, + timeout=google.api_core.gapic_v1.method.DEFAULT, + metadata=None, + ): + """ + Reads rows from the stream in the format prescribed by the ReadSession. + Each response contains one or more table rows, up to a maximum of 100 MiB + per response; read requests which attempt to read individual rows larger + than 100 MiB will fail. + + Each request also returns a set of stream statistics reflecting the current + state of the stream. + + Example: + >>> from google.cloud import bigquery_storage_v1beta2 + >>> + >>> client = bigquery_storage_v1beta2.BigQueryReadClient() + >>> + >>> for element in client.read_rows(): + ... # process element + ... pass + + Args: + read_stream (str): Required. Stream to read rows from. + offset (long): The offset requested must be less than the last row read from Read. + Requesting a larger offset is undefined. If not specified, start reading + from offset zero. + retry (Optional[google.api_core.retry.Retry]): A retry object used + to retry requests. If ``None`` is specified, requests will + be retried using a default configuration. + timeout (Optional[float]): The amount of time, in seconds, to wait + for the request to complete. Note that if ``retry`` is + specified, the timeout applies to each individual attempt. + metadata (Optional[Sequence[Tuple[str, str]]]): Additional metadata + that is provided to the method. + + Returns: + Iterable[~google.cloud.bigquery_storage_v1beta2.types.ReadRowsResponse]. + + Raises: + google.api_core.exceptions.GoogleAPICallError: If the request + failed for any reason. + google.api_core.exceptions.RetryError: If the request failed due + to a retryable error and retry attempts failed. + ValueError: If the parameters are invalid. + """ + # Wrap the transport method to add retry and timeout logic. + if "read_rows" not in self._inner_api_calls: + self._inner_api_calls[ + "read_rows" + ] = google.api_core.gapic_v1.method.wrap_method( + self.transport.read_rows, + default_retry=self._method_configs["ReadRows"].retry, + default_timeout=self._method_configs["ReadRows"].timeout, + client_info=self._client_info, + ) + + request = storage_pb2.ReadRowsRequest(read_stream=read_stream, offset=offset) + if metadata is None: + metadata = [] + metadata = list(metadata) + try: + routing_header = [("read_stream", read_stream)] + except AttributeError: + pass + else: + routing_metadata = google.api_core.gapic_v1.routing_header.to_grpc_metadata( + routing_header + ) + metadata.append(routing_metadata) # pragma: no cover + + return self._inner_api_calls["read_rows"]( + request, retry=retry, timeout=timeout, metadata=metadata + ) + + def split_read_stream( + self, + name=None, + fraction=None, + retry=google.api_core.gapic_v1.method.DEFAULT, + timeout=google.api_core.gapic_v1.method.DEFAULT, + metadata=None, + ): + """ + Splits a given ``ReadStream`` into two ``ReadStream`` objects. These + ``ReadStream`` objects are referred to as the primary and the residual + streams of the split. The original ``ReadStream`` can still be read from + in the same manner as before. Both of the returned ``ReadStream`` + objects can also be read from, and the rows returned by both child + streams will be the same as the rows read from the original stream. + + Moreover, the two child streams will be allocated back-to-back in the + original ``ReadStream``. Concretely, it is guaranteed that for streams + original, primary, and residual, that original[0-j] = primary[0-j] and + original[j-n] = residual[0-m] once the streams have been read to + completion. + + Example: + >>> from google.cloud import bigquery_storage_v1beta2 + >>> + >>> client = bigquery_storage_v1beta2.BigQueryReadClient() + >>> + >>> response = client.split_read_stream() + + Args: + name (str): Required. Name of the stream to split. + fraction (float): A value in the range (0.0, 1.0) that specifies the fractional point at + which the original stream should be split. The actual split point is + evaluated on pre-filtered rows, so if a filter is provided, then there is + no guarantee that the division of the rows between the new child streams + will be proportional to this fractional value. Additionally, because the + server-side unit for assigning data is collections of rows, this fraction + will always map to a data storage boundary on the server side. + retry (Optional[google.api_core.retry.Retry]): A retry object used + to retry requests. If ``None`` is specified, requests will + be retried using a default configuration. + timeout (Optional[float]): The amount of time, in seconds, to wait + for the request to complete. Note that if ``retry`` is + specified, the timeout applies to each individual attempt. + metadata (Optional[Sequence[Tuple[str, str]]]): Additional metadata + that is provided to the method. + + Returns: + A :class:`~google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamResponse` instance. + + Raises: + google.api_core.exceptions.GoogleAPICallError: If the request + failed for any reason. + google.api_core.exceptions.RetryError: If the request failed due + to a retryable error and retry attempts failed. + ValueError: If the parameters are invalid. + """ + # Wrap the transport method to add retry and timeout logic. + if "split_read_stream" not in self._inner_api_calls: + self._inner_api_calls[ + "split_read_stream" + ] = google.api_core.gapic_v1.method.wrap_method( + self.transport.split_read_stream, + default_retry=self._method_configs["SplitReadStream"].retry, + default_timeout=self._method_configs["SplitReadStream"].timeout, + client_info=self._client_info, + ) + + request = storage_pb2.SplitReadStreamRequest(name=name, fraction=fraction) + if metadata is None: + metadata = [] + metadata = list(metadata) + try: + routing_header = [("name", name)] + except AttributeError: + pass + else: + routing_metadata = google.api_core.gapic_v1.routing_header.to_grpc_metadata( + routing_header + ) + metadata.append(routing_metadata) # pragma: no cover + + return self._inner_api_calls["split_read_stream"]( + request, retry=retry, timeout=timeout, metadata=metadata + ) diff --git a/google/cloud/bigquery_storage_v1beta2/gapic/big_query_read_client_config.py b/google/cloud/bigquery_storage_v1beta2/gapic/big_query_read_client_config.py new file mode 100644 index 00000000..9915b24b --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/gapic/big_query_read_client_config.py @@ -0,0 +1,57 @@ +config = { + "interfaces": { + "google.cloud.bigquery.storage.v1beta2.BigQueryRead": { + "retry_codes": { + "idempotent": ["DEADLINE_EXCEEDED", "UNAVAILABLE"], + "unary_streaming": ["UNAVAILABLE"], + "non_idempotent": [], + }, + "retry_params": { + "default": { + "initial_retry_delay_millis": 100, + "retry_delay_multiplier": 1.3, + "max_retry_delay_millis": 60000, + "initial_rpc_timeout_millis": 20000, + "rpc_timeout_multiplier": 1.0, + "max_rpc_timeout_millis": 20000, + "total_timeout_millis": 600000, + }, + "create_read_session": { + "initial_retry_delay_millis": 100, + "retry_delay_multiplier": 1.3, + "max_retry_delay_millis": 60000, + "initial_rpc_timeout_millis": 120000, + "rpc_timeout_multiplier": 1.0, + "max_rpc_timeout_millis": 120000, + "total_timeout_millis": 600000, + }, + "read_rows": { + "initial_retry_delay_millis": 100, + "retry_delay_multiplier": 1.3, + "max_retry_delay_millis": 60000, + "initial_rpc_timeout_millis": 86400000, + "rpc_timeout_multiplier": 1.0, + "max_rpc_timeout_millis": 86400000, + "total_timeout_millis": 86400000, + }, + }, + "methods": { + "CreateReadSession": { + "timeout_millis": 120000, + "retry_codes_name": "idempotent", + "retry_params_name": "create_read_session", + }, + "ReadRows": { + "timeout_millis": 21600000, + "retry_codes_name": "unary_streaming", + "retry_params_name": "read_rows", + }, + "SplitReadStream": { + "timeout_millis": 120000, + "retry_codes_name": "idempotent", + "retry_params_name": "default", + }, + }, + } + } +} diff --git a/google/cloud/bigquery_storage_v1beta2/gapic/enums.py b/google/cloud/bigquery_storage_v1beta2/gapic/enums.py new file mode 100644 index 00000000..1e51229d --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/gapic/enums.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Wrappers for protocol buffer enum types.""" + +import enum + + +class DataFormat(enum.IntEnum): + """ + Data format for input or output data. + + Attributes: + DATA_FORMAT_UNSPECIFIED (int) + AVRO (int): Avro is a standard open source row based file format. + See https://avro.apache.org/ for more details. + ARROW (int): Arrow is a standard open source column-based message format. + See https://arrow.apache.org/ for more details. + """ + + DATA_FORMAT_UNSPECIFIED = 0 + AVRO = 1 + ARROW = 2 + + +class ArrowSerializationOptions(object): + class Format(enum.IntEnum): + """ + The IPC format to use when serializing Arrow streams. + + Attributes: + FORMAT_UNSPECIFIED (int): If unspecied the IPC format as of 0.15 release will be used. + ARROW_0_14 (int): Use the legacy IPC message format as of Apache Arrow Release 0.14. + ARROW_0_15 (int): Use the message format as of Apache Arrow Release 0.15. + """ + + FORMAT_UNSPECIFIED = 0 + ARROW_0_14 = 1 + ARROW_0_15 = 2 diff --git a/google/cloud/bigquery_storage_v1beta2/gapic/transports/__init__.py b/google/cloud/bigquery_storage_v1beta2/gapic/transports/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/google/cloud/bigquery_storage_v1beta2/gapic/transports/big_query_read_grpc_transport.py b/google/cloud/bigquery_storage_v1beta2/gapic/transports/big_query_read_grpc_transport.py new file mode 100644 index 00000000..b2c4f09c --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/gapic/transports/big_query_read_grpc_transport.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import google.api_core.grpc_helpers + +from google.cloud.bigquery_storage_v1beta2.proto import storage_pb2_grpc + + +class BigQueryReadGrpcTransport(object): + """gRPC transport class providing stubs for + google.cloud.bigquery.storage.v1beta2 BigQueryRead API. + + The transport provides access to the raw gRPC stubs, + which can be used to take advantage of advanced + features of gRPC. + """ + + # The scopes needed to make gRPC calls to all of the methods defined + # in this service. + _OAUTH_SCOPES = ( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.readonly", + "https://www.googleapis.com/auth/cloud-platform", + ) + + def __init__( + self, + channel=None, + credentials=None, + address="bigquerystorage.googleapis.com:443", + ): + """Instantiate the transport class. + + Args: + channel (grpc.Channel): A ``Channel`` instance through + which to make calls. This argument is mutually exclusive + with ``credentials``; providing both will raise an exception. + credentials (google.auth.credentials.Credentials): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If none + are specified, the client will attempt to ascertain the + credentials from the environment. + address (str): The address where the service is hosted. + """ + # If both `channel` and `credentials` are specified, raise an + # exception (channels come with credentials baked in already). + if channel is not None and credentials is not None: # pragma: no cover + raise ValueError( + "The `channel` and `credentials` arguments are mutually " "exclusive." + ) + + # Create the channel. + if channel is None: # pragma: no cover + channel = self.create_channel( + address=address, + credentials=credentials, + options={ + "grpc.max_send_message_length": -1, + "grpc.max_receive_message_length": -1, + }.items(), + ) + + self._channel = channel + + # gRPC uses objects called "stubs" that are bound to the + # channel and provide a basic method for each RPC. + self._stubs = { + "big_query_read_stub": storage_pb2_grpc.BigQueryReadStub(channel) + } + + @classmethod + def create_channel( + cls, address="bigquerystorage.googleapis.com:443", credentials=None, **kwargs + ): + """Create and return a gRPC channel object. + + Args: + address (str): The host for the channel to use. + credentials (~.Credentials): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + kwargs (dict): Keyword arguments, which are passed to the + channel creation. + + Returns: + grpc.Channel: A gRPC channel object. + """ + return google.api_core.grpc_helpers.create_channel( # pragma: no cover + address, credentials=credentials, scopes=cls._OAUTH_SCOPES, **kwargs + ) + + @property + def channel(self): + """The gRPC channel used by the transport. + + Returns: + grpc.Channel: A gRPC channel object. + """ + return self._channel + + @property + def create_read_session(self): + """Return the gRPC stub for :meth:`BigQueryReadClient.create_read_session`. + + Creates a new read session. A read session divides the contents of a + BigQuery table into one or more streams, which can then be used to read + data from the table. The read session also specifies properties of the + data to be read, such as a list of columns or a push-down filter describing + the rows to be returned. + + A particular row can be read by at most one stream. When the caller has + reached the end of each stream in the session, then all the data in the + table has been read. + + Data is assigned to each stream such that roughly the same number of + rows can be read from each stream. Because the server-side unit for + assigning data is collections of rows, the API does not guarantee that + each stream will return the same number or rows. Additionally, the + limits are enforced based on the number of pre-filtered rows, so some + filters can lead to lopsided assignments. + + Read sessions automatically expire 24 hours after they are created and do + not require manual clean-up by the caller. + + Returns: + Callable: A callable which accepts the appropriate + deserialized request object and returns a + deserialized response object. + """ + return self._stubs["big_query_read_stub"].CreateReadSession + + @property + def read_rows(self): + """Return the gRPC stub for :meth:`BigQueryReadClient.read_rows`. + + Reads rows from the stream in the format prescribed by the ReadSession. + Each response contains one or more table rows, up to a maximum of 100 MiB + per response; read requests which attempt to read individual rows larger + than 100 MiB will fail. + + Each request also returns a set of stream statistics reflecting the current + state of the stream. + + Returns: + Callable: A callable which accepts the appropriate + deserialized request object and returns a + deserialized response object. + """ + return self._stubs["big_query_read_stub"].ReadRows + + @property + def split_read_stream(self): + """Return the gRPC stub for :meth:`BigQueryReadClient.split_read_stream`. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. These + ``ReadStream`` objects are referred to as the primary and the residual + streams of the split. The original ``ReadStream`` can still be read from + in the same manner as before. Both of the returned ``ReadStream`` + objects can also be read from, and the rows returned by both child + streams will be the same as the rows read from the original stream. + + Moreover, the two child streams will be allocated back-to-back in the + original ``ReadStream``. Concretely, it is guaranteed that for streams + original, primary, and residual, that original[0-j] = primary[0-j] and + original[j-n] = residual[0-m] once the streams have been read to + completion. + + Returns: + Callable: A callable which accepts the appropriate + deserialized request object and returns a + deserialized response object. + """ + return self._stubs["big_query_read_stub"].SplitReadStream diff --git a/google/cloud/bigquery_storage_v1beta2/proto/__init__.py b/google/cloud/bigquery_storage_v1beta2/proto/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto b/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto new file mode 100644 index 00000000..ad178dce --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto @@ -0,0 +1,58 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.cloud.bigquery.storage.v1beta2; + +option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storage"; +option java_multiple_files = true; +option java_outer_classname = "ArrowProto"; +option java_package = "com.google.cloud.bigquery.storage.v1beta2"; + +// Arrow schema as specified in +// https://arrow.apache.org/docs/python/api/datatypes.html +// and serialized to bytes using IPC: +// https://arrow.apache.org/docs/ipc.html. +// +// See code samples on how this message can be deserialized. +message ArrowSchema { + // IPC serialized Arrow schema. + bytes serialized_schema = 1; +} + +// Arrow RecordBatch. +message ArrowRecordBatch { + // IPC-serialized Arrow RecordBatch. + bytes serialized_record_batch = 1; +} + +// Contains options specific to Arrow Serialization. +message ArrowSerializationOptions { + // The IPC format to use when serializing Arrow streams. + enum Format { + // If unspecied the IPC format as of 0.15 release will be used. + FORMAT_UNSPECIFIED = 0; + + // Use the legacy IPC message format as of Apache Arrow Release 0.14. + ARROW_0_14 = 1; + + // Use the message format as of Apache Arrow Release 0.15. + ARROW_0_15 = 2; + } + + // The Arrow IPC format to use. + Format format = 1; +} diff --git a/google/cloud/bigquery_storage_v1beta2/proto/arrow_pb2.py b/google/cloud/bigquery_storage_v1beta2/proto/arrow_pb2.py new file mode 100644 index 00000000..ecf172c3 --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/arrow_pb2.py @@ -0,0 +1,246 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: google/cloud/bigquery/storage_v1beta2/proto/arrow.proto + +import sys + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="google/cloud/bigquery/storage_v1beta2/proto/arrow.proto", + package="google.cloud.bigquery.storage.v1beta2", + syntax="proto3", + serialized_options=_b( + "\n)com.google.cloud.bigquery.storage.v1beta2B\nArrowProtoP\001ZLgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storage" + ), + serialized_pb=_b( + '\n7google/cloud/bigquery/storage_v1beta2/proto/arrow.proto\x12%google.cloud.bigquery.storage.v1beta2"(\n\x0b\x41rrowSchema\x12\x19\n\x11serialized_schema\x18\x01 \x01(\x0c"3\n\x10\x41rrowRecordBatch\x12\x1f\n\x17serialized_record_batch\x18\x01 \x01(\x0c"\xb6\x01\n\x19\x41rrowSerializationOptions\x12W\n\x06\x66ormat\x18\x01 \x01(\x0e\x32G.google.cloud.bigquery.storage.v1beta2.ArrowSerializationOptions.Format"@\n\x06\x46ormat\x12\x16\n\x12\x46ORMAT_UNSPECIFIED\x10\x00\x12\x0e\n\nARROW_0_14\x10\x01\x12\x0e\n\nARROW_0_15\x10\x02\x42\x87\x01\n)com.google.cloud.bigquery.storage.v1beta2B\nArrowProtoP\x01ZLgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storageb\x06proto3' + ), +) + + +_ARROWSERIALIZATIONOPTIONS_FORMAT = _descriptor.EnumDescriptor( + name="Format", + full_name="google.cloud.bigquery.storage.v1beta2.ArrowSerializationOptions.Format", + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name="FORMAT_UNSPECIFIED", + index=0, + number=0, + serialized_options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name="ARROW_0_14", index=1, number=1, serialized_options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ARROW_0_15", index=2, number=2, serialized_options=None, type=None + ), + ], + containing_type=None, + serialized_options=None, + serialized_start=312, + serialized_end=376, +) +_sym_db.RegisterEnumDescriptor(_ARROWSERIALIZATIONOPTIONS_FORMAT) + + +_ARROWSCHEMA = _descriptor.Descriptor( + name="ArrowSchema", + full_name="google.cloud.bigquery.storage.v1beta2.ArrowSchema", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="serialized_schema", + full_name="google.cloud.bigquery.storage.v1beta2.ArrowSchema.serialized_schema", + index=0, + number=1, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=98, + serialized_end=138, +) + + +_ARROWRECORDBATCH = _descriptor.Descriptor( + name="ArrowRecordBatch", + full_name="google.cloud.bigquery.storage.v1beta2.ArrowRecordBatch", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="serialized_record_batch", + full_name="google.cloud.bigquery.storage.v1beta2.ArrowRecordBatch.serialized_record_batch", + index=0, + number=1, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=140, + serialized_end=191, +) + + +_ARROWSERIALIZATIONOPTIONS = _descriptor.Descriptor( + name="ArrowSerializationOptions", + full_name="google.cloud.bigquery.storage.v1beta2.ArrowSerializationOptions", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="format", + full_name="google.cloud.bigquery.storage.v1beta2.ArrowSerializationOptions.format", + index=0, + number=1, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[_ARROWSERIALIZATIONOPTIONS_FORMAT], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=194, + serialized_end=376, +) + +_ARROWSERIALIZATIONOPTIONS.fields_by_name[ + "format" +].enum_type = _ARROWSERIALIZATIONOPTIONS_FORMAT +_ARROWSERIALIZATIONOPTIONS_FORMAT.containing_type = _ARROWSERIALIZATIONOPTIONS +DESCRIPTOR.message_types_by_name["ArrowSchema"] = _ARROWSCHEMA +DESCRIPTOR.message_types_by_name["ArrowRecordBatch"] = _ARROWRECORDBATCH +DESCRIPTOR.message_types_by_name[ + "ArrowSerializationOptions" +] = _ARROWSERIALIZATIONOPTIONS +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +ArrowSchema = _reflection.GeneratedProtocolMessageType( + "ArrowSchema", + (_message.Message,), + dict( + DESCRIPTOR=_ARROWSCHEMA, + __module__="google.cloud.bigquery.storage_v1beta2.proto.arrow_pb2", + __doc__="""Arrow schema as specified in + https://arrow.apache.org/docs/python/api/datatypes.html and serialized + to bytes using IPC: https://arrow.apache.org/docs/ipc.html. + + See code samples on how this message can be deserialized. + + + Attributes: + serialized_schema: + IPC serialized Arrow schema. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ArrowSchema) + ), +) +_sym_db.RegisterMessage(ArrowSchema) + +ArrowRecordBatch = _reflection.GeneratedProtocolMessageType( + "ArrowRecordBatch", + (_message.Message,), + dict( + DESCRIPTOR=_ARROWRECORDBATCH, + __module__="google.cloud.bigquery.storage_v1beta2.proto.arrow_pb2", + __doc__="""Arrow RecordBatch. + + + Attributes: + serialized_record_batch: + IPC-serialized Arrow RecordBatch. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ArrowRecordBatch) + ), +) +_sym_db.RegisterMessage(ArrowRecordBatch) + +ArrowSerializationOptions = _reflection.GeneratedProtocolMessageType( + "ArrowSerializationOptions", + (_message.Message,), + dict( + DESCRIPTOR=_ARROWSERIALIZATIONOPTIONS, + __module__="google.cloud.bigquery.storage_v1beta2.proto.arrow_pb2", + __doc__="""Contains options specific to Arrow Serialization. + + + Attributes: + format: + The Arrow IPC format to use. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ArrowSerializationOptions) + ), +) +_sym_db.RegisterMessage(ArrowSerializationOptions) + + +DESCRIPTOR._options = None +# @@protoc_insertion_point(module_scope) diff --git a/google/cloud/bigquery_storage_v1beta2/proto/arrow_pb2_grpc.py b/google/cloud/bigquery_storage_v1beta2/proto/arrow_pb2_grpc.py new file mode 100644 index 00000000..07cb78fe --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/arrow_pb2_grpc.py @@ -0,0 +1,2 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc diff --git a/google/cloud/bigquery_storage_v1beta2/proto/avro.proto b/google/cloud/bigquery_storage_v1beta2/proto/avro.proto new file mode 100644 index 00000000..37194690 --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/avro.proto @@ -0,0 +1,36 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.cloud.bigquery.storage.v1beta2; + +option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storage"; +option java_multiple_files = true; +option java_outer_classname = "AvroProto"; +option java_package = "com.google.cloud.bigquery.storage.v1beta2"; + +// Avro schema. +message AvroSchema { + // Json serialized schema, as described at + // https://avro.apache.org/docs/1.8.1/spec.html. + string schema = 1; +} + +// Avro rows. +message AvroRows { + // Binary serialized rows in a block. + bytes serialized_binary_rows = 1; +} diff --git a/google/cloud/bigquery_storage_v1beta2/proto/avro_pb2.py b/google/cloud/bigquery_storage_v1beta2/proto/avro_pb2.py new file mode 100644 index 00000000..f3df85cf --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/avro_pb2.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: google/cloud/bigquery/storage_v1beta2/proto/avro.proto + +import sys + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="google/cloud/bigquery/storage_v1beta2/proto/avro.proto", + package="google.cloud.bigquery.storage.v1beta2", + syntax="proto3", + serialized_options=_b( + "\n)com.google.cloud.bigquery.storage.v1beta2B\tAvroProtoP\001ZLgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storage" + ), + serialized_pb=_b( + '\n6google/cloud/bigquery/storage_v1beta2/proto/avro.proto\x12%google.cloud.bigquery.storage.v1beta2"\x1c\n\nAvroSchema\x12\x0e\n\x06schema\x18\x01 \x01(\t"*\n\x08\x41vroRows\x12\x1e\n\x16serialized_binary_rows\x18\x01 \x01(\x0c\x42\x86\x01\n)com.google.cloud.bigquery.storage.v1beta2B\tAvroProtoP\x01ZLgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storageb\x06proto3' + ), +) + + +_AVROSCHEMA = _descriptor.Descriptor( + name="AvroSchema", + full_name="google.cloud.bigquery.storage.v1beta2.AvroSchema", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="schema", + full_name="google.cloud.bigquery.storage.v1beta2.AvroSchema.schema", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=97, + serialized_end=125, +) + + +_AVROROWS = _descriptor.Descriptor( + name="AvroRows", + full_name="google.cloud.bigquery.storage.v1beta2.AvroRows", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="serialized_binary_rows", + full_name="google.cloud.bigquery.storage.v1beta2.AvroRows.serialized_binary_rows", + index=0, + number=1, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=127, + serialized_end=169, +) + +DESCRIPTOR.message_types_by_name["AvroSchema"] = _AVROSCHEMA +DESCRIPTOR.message_types_by_name["AvroRows"] = _AVROROWS +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +AvroSchema = _reflection.GeneratedProtocolMessageType( + "AvroSchema", + (_message.Message,), + dict( + DESCRIPTOR=_AVROSCHEMA, + __module__="google.cloud.bigquery.storage_v1beta2.proto.avro_pb2", + __doc__="""Avro schema. + + + Attributes: + schema: + Json serialized schema, as described at + https://avro.apache.org/docs/1.8.1/spec.html. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.AvroSchema) + ), +) +_sym_db.RegisterMessage(AvroSchema) + +AvroRows = _reflection.GeneratedProtocolMessageType( + "AvroRows", + (_message.Message,), + dict( + DESCRIPTOR=_AVROROWS, + __module__="google.cloud.bigquery.storage_v1beta2.proto.avro_pb2", + __doc__="""Avro rows. + + + Attributes: + serialized_binary_rows: + Binary serialized rows in a block. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.AvroRows) + ), +) +_sym_db.RegisterMessage(AvroRows) + + +DESCRIPTOR._options = None +# @@protoc_insertion_point(module_scope) diff --git a/google/cloud/bigquery_storage_v1beta2/proto/avro_pb2_grpc.py b/google/cloud/bigquery_storage_v1beta2/proto/avro_pb2_grpc.py new file mode 100644 index 00000000..07cb78fe --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/avro_pb2_grpc.py @@ -0,0 +1,2 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc diff --git a/google/cloud/bigquery_storage_v1beta2/proto/storage.proto b/google/cloud/bigquery_storage_v1beta2/proto/storage.proto new file mode 100644 index 00000000..373c8352 --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/storage.proto @@ -0,0 +1,230 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.cloud.bigquery.storage.v1beta2; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/resource.proto"; +import "google/cloud/bigquery/storage/v1beta2/arrow.proto"; +import "google/cloud/bigquery/storage/v1beta2/avro.proto"; +import "google/cloud/bigquery/storage/v1beta2/stream.proto"; + +option go_package = "google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storage"; +option java_multiple_files = true; +option java_outer_classname = "StorageProto"; +option java_package = "com.google.cloud.bigquery.storage.v1beta2"; + +// BigQuery Read API. +// +// The Read API can be used to read data from BigQuery. +service BigQueryRead { + option (google.api.default_host) = "bigquerystorage.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/bigquery," + "https://www.googleapis.com/auth/bigquery.readonly," + "https://www.googleapis.com/auth/cloud-platform"; + + // Creates a new read session. A read session divides the contents of a + // BigQuery table into one or more streams, which can then be used to read + // data from the table. The read session also specifies properties of the + // data to be read, such as a list of columns or a push-down filter describing + // the rows to be returned. + // + // A particular row can be read by at most one stream. When the caller has + // reached the end of each stream in the session, then all the data in the + // table has been read. + // + // Data is assigned to each stream such that roughly the same number of + // rows can be read from each stream. Because the server-side unit for + // assigning data is collections of rows, the API does not guarantee that + // each stream will return the same number or rows. Additionally, the + // limits are enforced based on the number of pre-filtered rows, so some + // filters can lead to lopsided assignments. + // + // Read sessions automatically expire 24 hours after they are created and do + // not require manual clean-up by the caller. + rpc CreateReadSession(CreateReadSessionRequest) returns (ReadSession) { + option (google.api.http) = { + post: "/v1beta2/{read_session.table=projects/*/datasets/*/tables/*}" + body: "*" + }; + option (google.api.method_signature) = "parent,read_session,max_stream_count"; + } + + // Reads rows from the stream in the format prescribed by the ReadSession. + // Each response contains one or more table rows, up to a maximum of 100 MiB + // per response; read requests which attempt to read individual rows larger + // than 100 MiB will fail. + // + // Each request also returns a set of stream statistics reflecting the current + // state of the stream. + rpc ReadRows(ReadRowsRequest) returns (stream ReadRowsResponse) { + option (google.api.http) = { + get: "/v1beta2/{read_stream=projects/*/locations/*/sessions/*/streams/*}" + }; + option (google.api.method_signature) = "read_stream,offset"; + } + + // Splits a given `ReadStream` into two `ReadStream` objects. These + // `ReadStream` objects are referred to as the primary and the residual + // streams of the split. The original `ReadStream` can still be read from in + // the same manner as before. Both of the returned `ReadStream` objects can + // also be read from, and the rows returned by both child streams will be + // the same as the rows read from the original stream. + // + // Moreover, the two child streams will be allocated back-to-back in the + // original `ReadStream`. Concretely, it is guaranteed that for streams + // original, primary, and residual, that original[0-j] = primary[0-j] and + // original[j-n] = residual[0-m] once the streams have been read to + // completion. + rpc SplitReadStream(SplitReadStreamRequest) returns (SplitReadStreamResponse) { + option (google.api.http) = { + get: "/v1beta2/{name=projects/*/locations/*/sessions/*/streams/*}" + }; + } +} + +// Request message for `CreateReadSession`. +message CreateReadSessionRequest { + // Required. The request project that owns the session, in the form of + // `projects/{project_id}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "cloudresourcemanager.googleapis.com/Project" + } + ]; + + // Required. Session to be created. + ReadSession read_session = 2 [(google.api.field_behavior) = REQUIRED]; + + // Max initial number of streams. If unset or zero, the server will + // provide a value of streams so as to produce reasonable throughput. Must be + // non-negative. The number of streams may be lower than the requested number, + // depending on the amount parallelism that is reasonable for the table. Error + // will be returned if the max count is greater than the current system + // max limit of 1,000. + // + // Streams must be read starting from offset 0. + int32 max_stream_count = 3; +} + +// Request message for `ReadRows`. +message ReadRowsRequest { + // Required. Stream to read rows from. + string read_stream = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquerystorage.googleapis.com/ReadStream" + } + ]; + + // The offset requested must be less than the last row read from Read. + // Requesting a larger offset is undefined. If not specified, start reading + // from offset zero. + int64 offset = 2; +} + +// Information on if the current connection is being throttled. +message ThrottleState { + // How much this connection is being throttled. Zero means no throttling, + // 100 means fully throttled. + int32 throttle_percent = 1; +} + +// Estimated stream statistics for a given Stream. +message StreamStats { + message Progress { + // The fraction of rows assigned to the stream that have been processed by + // the server so far, not including the rows in the current response + // message. + // + // This value, along with `at_response_end`, can be used to interpolate + // the progress made as the rows in the message are being processed using + // the following formula: `at_response_start + (at_response_end - + // at_response_start) * rows_processed_from_response / rows_in_response`. + // + // Note that if a filter is provided, the `at_response_end` value of the + // previous response may not necessarily be equal to the + // `at_response_start` value of the current response. + double at_response_start = 1; + + // Similar to `at_response_start`, except that this value includes the + // rows in the current response. + double at_response_end = 2; + } + + // Represents the progress of the current stream. + Progress progress = 2; +} + +// Response from calling `ReadRows` may include row data, progress and +// throttling information. +message ReadRowsResponse { + // Row data is returned in format specified during session creation. + oneof rows { + // Serialized row data in AVRO format. + AvroRows avro_rows = 3; + + // Serialized row data in Arrow RecordBatch format. + ArrowRecordBatch arrow_record_batch = 4; + } + + // Number of serialized rows in the rows block. + int64 row_count = 6; + + // Statistics for the stream. + StreamStats stats = 2; + + // Throttling state. If unset, the latest response still describes + // the current throttling status. + ThrottleState throttle_state = 5; +} + +// Request message for `SplitReadStream`. +message SplitReadStreamRequest { + // Required. Name of the stream to split. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquerystorage.googleapis.com/ReadStream" + } + ]; + + // A value in the range (0.0, 1.0) that specifies the fractional point at + // which the original stream should be split. The actual split point is + // evaluated on pre-filtered rows, so if a filter is provided, then there is + // no guarantee that the division of the rows between the new child streams + // will be proportional to this fractional value. Additionally, because the + // server-side unit for assigning data is collections of rows, this fraction + // will always map to a data storage boundary on the server side. + double fraction = 2; +} + +// Response message for `SplitReadStream`. +message SplitReadStreamResponse { + // Primary stream, which contains the beginning portion of + // |original_stream|. An empty value indicates that the original stream can no + // longer be split. + ReadStream primary_stream = 1; + + // Remainder stream, which contains the tail of |original_stream|. An empty + // value indicates that the original stream can no longer be split. + ReadStream remainder_stream = 2; +} diff --git a/google/cloud/bigquery_storage_v1beta2/proto/storage_pb2.py b/google/cloud/bigquery_storage_v1beta2/proto/storage_pb2.py new file mode 100644 index 00000000..d93e2e0b --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/storage_pb2.py @@ -0,0 +1,867 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: google/cloud/bigquery/storage_v1beta2/proto/storage.proto + +import sys + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from google.api import client_pb2 as google_dot_api_dot_client__pb2 +from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 +from google.api import resource_pb2 as google_dot_api_dot_resource__pb2 +from google.cloud.bigquery_storage_v1beta2.proto import ( + arrow_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_arrow__pb2, +) +from google.cloud.bigquery_storage_v1beta2.proto import ( + avro_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_avro__pb2, +) +from google.cloud.bigquery_storage_v1beta2.proto import ( + stream_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_stream__pb2, +) + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="google/cloud/bigquery/storage_v1beta2/proto/storage.proto", + package="google.cloud.bigquery.storage.v1beta2", + syntax="proto3", + serialized_options=_b( + "\n)com.google.cloud.bigquery.storage.v1beta2B\014StorageProtoP\001ZLgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storage" + ), + serialized_pb=_b( + '\n9google/cloud/bigquery/storage_v1beta2/proto/storage.proto\x12%google.cloud.bigquery.storage.v1beta2\x1a\x1cgoogle/api/annotations.proto\x1a\x17google/api/client.proto\x1a\x1fgoogle/api/field_behavior.proto\x1a\x19google/api/resource.proto\x1a\x37google/cloud/bigquery/storage_v1beta2/proto/arrow.proto\x1a\x36google/cloud/bigquery/storage_v1beta2/proto/avro.proto\x1a\x38google/cloud/bigquery/storage_v1beta2/proto/stream.proto"\xc8\x01\n\x18\x43reateReadSessionRequest\x12\x43\n\x06parent\x18\x01 \x01(\tB3\xe0\x41\x02\xfa\x41-\n+cloudresourcemanager.googleapis.com/Project\x12M\n\x0cread_session\x18\x02 \x01(\x0b\x32\x32.google.cloud.bigquery.storage.v1beta2.ReadSessionB\x03\xe0\x41\x02\x12\x18\n\x10max_stream_count\x18\x03 \x01(\x05"i\n\x0fReadRowsRequest\x12\x46\n\x0bread_stream\x18\x01 \x01(\tB1\xe0\x41\x02\xfa\x41+\n)bigquerystorage.googleapis.com/ReadStream\x12\x0e\n\x06offset\x18\x02 \x01(\x03")\n\rThrottleState\x12\x18\n\x10throttle_percent\x18\x01 \x01(\x05"\x9c\x01\n\x0bStreamStats\x12M\n\x08progress\x18\x02 \x01(\x0b\x32;.google.cloud.bigquery.storage.v1beta2.StreamStats.Progress\x1a>\n\x08Progress\x12\x19\n\x11\x61t_response_start\x18\x01 \x01(\x01\x12\x17\n\x0f\x61t_response_end\x18\x02 \x01(\x01"\xdb\x02\n\x10ReadRowsResponse\x12\x44\n\tavro_rows\x18\x03 \x01(\x0b\x32/.google.cloud.bigquery.storage.v1beta2.AvroRowsH\x00\x12U\n\x12\x61rrow_record_batch\x18\x04 \x01(\x0b\x32\x37.google.cloud.bigquery.storage.v1beta2.ArrowRecordBatchH\x00\x12\x11\n\trow_count\x18\x06 \x01(\x03\x12\x41\n\x05stats\x18\x02 \x01(\x0b\x32\x32.google.cloud.bigquery.storage.v1beta2.StreamStats\x12L\n\x0ethrottle_state\x18\x05 \x01(\x0b\x32\x34.google.cloud.bigquery.storage.v1beta2.ThrottleStateB\x06\n\x04rows"k\n\x16SplitReadStreamRequest\x12?\n\x04name\x18\x01 \x01(\tB1\xe0\x41\x02\xfa\x41+\n)bigquerystorage.googleapis.com/ReadStream\x12\x10\n\x08\x66raction\x18\x02 \x01(\x01"\xb1\x01\n\x17SplitReadStreamResponse\x12I\n\x0eprimary_stream\x18\x01 \x01(\x0b\x32\x31.google.cloud.bigquery.storage.v1beta2.ReadStream\x12K\n\x10remainder_stream\x18\x02 \x01(\x0b\x32\x31.google.cloud.bigquery.storage.v1beta2.ReadStream2\xf3\x06\n\x0c\x42igQueryRead\x12\xf8\x01\n\x11\x43reateReadSession\x12?.google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest\x1a\x32.google.cloud.bigquery.storage.v1beta2.ReadSession"n\x82\xd3\xe4\x93\x02\x41".google.cloud.bigquery.storage.v1beta2.SplitReadStreamResponse"C\x82\xd3\xe4\x93\x02=\x12;/v1beta2/{name=projects/*/locations/*/sessions/*/streams/*}\x1a\xae\x01\xca\x41\x1e\x62igquerystorage.googleapis.com\xd2\x41\x89\x01https://www.googleapis.com/auth/bigquery,https://www.googleapis.com/auth/bigquery.readonly,https://www.googleapis.com/auth/cloud-platformB\x89\x01\n)com.google.cloud.bigquery.storage.v1beta2B\x0cStorageProtoP\x01ZLgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storageb\x06proto3' + ), + dependencies=[ + google_dot_api_dot_annotations__pb2.DESCRIPTOR, + google_dot_api_dot_client__pb2.DESCRIPTOR, + google_dot_api_dot_field__behavior__pb2.DESCRIPTOR, + google_dot_api_dot_resource__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_arrow__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_avro__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_stream__pb2.DESCRIPTOR, + ], +) + + +_CREATEREADSESSIONREQUEST = _descriptor.Descriptor( + name="CreateReadSessionRequest", + full_name="google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="parent", + full_name="google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest.parent", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b( + "\340A\002\372A-\n+cloudresourcemanager.googleapis.com/Project" + ), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="read_session", + full_name="google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest.read_session", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\002"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="max_stream_count", + full_name="google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest.max_stream_count", + index=2, + number=3, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=387, + serialized_end=587, +) + + +_READROWSREQUEST = _descriptor.Descriptor( + name="ReadRowsRequest", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsRequest", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="read_stream", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsRequest.read_stream", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b( + "\340A\002\372A+\n)bigquerystorage.googleapis.com/ReadStream" + ), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="offset", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsRequest.offset", + index=1, + number=2, + type=3, + cpp_type=2, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=589, + serialized_end=694, +) + + +_THROTTLESTATE = _descriptor.Descriptor( + name="ThrottleState", + full_name="google.cloud.bigquery.storage.v1beta2.ThrottleState", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="throttle_percent", + full_name="google.cloud.bigquery.storage.v1beta2.ThrottleState.throttle_percent", + index=0, + number=1, + type=5, + cpp_type=1, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=696, + serialized_end=737, +) + + +_STREAMSTATS_PROGRESS = _descriptor.Descriptor( + name="Progress", + full_name="google.cloud.bigquery.storage.v1beta2.StreamStats.Progress", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="at_response_start", + full_name="google.cloud.bigquery.storage.v1beta2.StreamStats.Progress.at_response_start", + index=0, + number=1, + type=1, + cpp_type=5, + label=1, + has_default_value=False, + default_value=float(0), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="at_response_end", + full_name="google.cloud.bigquery.storage.v1beta2.StreamStats.Progress.at_response_end", + index=1, + number=2, + type=1, + cpp_type=5, + label=1, + has_default_value=False, + default_value=float(0), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=834, + serialized_end=896, +) + +_STREAMSTATS = _descriptor.Descriptor( + name="StreamStats", + full_name="google.cloud.bigquery.storage.v1beta2.StreamStats", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="progress", + full_name="google.cloud.bigquery.storage.v1beta2.StreamStats.progress", + index=0, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[_STREAMSTATS_PROGRESS], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=740, + serialized_end=896, +) + + +_READROWSRESPONSE = _descriptor.Descriptor( + name="ReadRowsResponse", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsResponse", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="avro_rows", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsResponse.avro_rows", + index=0, + number=3, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="arrow_record_batch", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsResponse.arrow_record_batch", + index=1, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="row_count", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsResponse.row_count", + index=2, + number=6, + type=3, + cpp_type=2, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="stats", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsResponse.stats", + index=3, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="throttle_state", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsResponse.throttle_state", + index=4, + number=5, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name="rows", + full_name="google.cloud.bigquery.storage.v1beta2.ReadRowsResponse.rows", + index=0, + containing_type=None, + fields=[], + ) + ], + serialized_start=899, + serialized_end=1246, +) + + +_SPLITREADSTREAMREQUEST = _descriptor.Descriptor( + name="SplitReadStreamRequest", + full_name="google.cloud.bigquery.storage.v1beta2.SplitReadStreamRequest", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="name", + full_name="google.cloud.bigquery.storage.v1beta2.SplitReadStreamRequest.name", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b( + "\340A\002\372A+\n)bigquerystorage.googleapis.com/ReadStream" + ), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="fraction", + full_name="google.cloud.bigquery.storage.v1beta2.SplitReadStreamRequest.fraction", + index=1, + number=2, + type=1, + cpp_type=5, + label=1, + has_default_value=False, + default_value=float(0), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=1248, + serialized_end=1355, +) + + +_SPLITREADSTREAMRESPONSE = _descriptor.Descriptor( + name="SplitReadStreamResponse", + full_name="google.cloud.bigquery.storage.v1beta2.SplitReadStreamResponse", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="primary_stream", + full_name="google.cloud.bigquery.storage.v1beta2.SplitReadStreamResponse.primary_stream", + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="remainder_stream", + full_name="google.cloud.bigquery.storage.v1beta2.SplitReadStreamResponse.remainder_stream", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=1358, + serialized_end=1535, +) + +_CREATEREADSESSIONREQUEST.fields_by_name[ + "read_session" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_stream__pb2._READSESSION +) +_STREAMSTATS_PROGRESS.containing_type = _STREAMSTATS +_STREAMSTATS.fields_by_name["progress"].message_type = _STREAMSTATS_PROGRESS +_READROWSRESPONSE.fields_by_name[ + "avro_rows" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_avro__pb2._AVROROWS +) +_READROWSRESPONSE.fields_by_name[ + "arrow_record_batch" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_arrow__pb2._ARROWRECORDBATCH +) +_READROWSRESPONSE.fields_by_name["stats"].message_type = _STREAMSTATS +_READROWSRESPONSE.fields_by_name["throttle_state"].message_type = _THROTTLESTATE +_READROWSRESPONSE.oneofs_by_name["rows"].fields.append( + _READROWSRESPONSE.fields_by_name["avro_rows"] +) +_READROWSRESPONSE.fields_by_name[ + "avro_rows" +].containing_oneof = _READROWSRESPONSE.oneofs_by_name["rows"] +_READROWSRESPONSE.oneofs_by_name["rows"].fields.append( + _READROWSRESPONSE.fields_by_name["arrow_record_batch"] +) +_READROWSRESPONSE.fields_by_name[ + "arrow_record_batch" +].containing_oneof = _READROWSRESPONSE.oneofs_by_name["rows"] +_SPLITREADSTREAMRESPONSE.fields_by_name[ + "primary_stream" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_stream__pb2._READSTREAM +) +_SPLITREADSTREAMRESPONSE.fields_by_name[ + "remainder_stream" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_stream__pb2._READSTREAM +) +DESCRIPTOR.message_types_by_name["CreateReadSessionRequest"] = _CREATEREADSESSIONREQUEST +DESCRIPTOR.message_types_by_name["ReadRowsRequest"] = _READROWSREQUEST +DESCRIPTOR.message_types_by_name["ThrottleState"] = _THROTTLESTATE +DESCRIPTOR.message_types_by_name["StreamStats"] = _STREAMSTATS +DESCRIPTOR.message_types_by_name["ReadRowsResponse"] = _READROWSRESPONSE +DESCRIPTOR.message_types_by_name["SplitReadStreamRequest"] = _SPLITREADSTREAMREQUEST +DESCRIPTOR.message_types_by_name["SplitReadStreamResponse"] = _SPLITREADSTREAMRESPONSE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +CreateReadSessionRequest = _reflection.GeneratedProtocolMessageType( + "CreateReadSessionRequest", + (_message.Message,), + dict( + DESCRIPTOR=_CREATEREADSESSIONREQUEST, + __module__="google.cloud.bigquery.storage_v1beta2.proto.storage_pb2", + __doc__="""Request message for ``CreateReadSession``. + + + Attributes: + parent: + Required. The request project that owns the session, in the + form of ``projects/{project_id}``. + read_session: + Required. Session to be created. + max_stream_count: + Max initial number of streams. If unset or zero, the server + will provide a value of streams so as to produce reasonable + throughput. Must be non-negative. The number of streams may be + lower than the requested number, depending on the amount + parallelism that is reasonable for the table. Error will be + returned if the max count is greater than the current system + max limit of 1,000. Streams must be read starting from offset + 0. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest) + ), +) +_sym_db.RegisterMessage(CreateReadSessionRequest) + +ReadRowsRequest = _reflection.GeneratedProtocolMessageType( + "ReadRowsRequest", + (_message.Message,), + dict( + DESCRIPTOR=_READROWSREQUEST, + __module__="google.cloud.bigquery.storage_v1beta2.proto.storage_pb2", + __doc__="""Request message for ``ReadRows``. + + + Attributes: + read_stream: + Required. Stream to read rows from. + offset: + The offset requested must be less than the last row read from + Read. Requesting a larger offset is undefined. If not + specified, start reading from offset zero. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ReadRowsRequest) + ), +) +_sym_db.RegisterMessage(ReadRowsRequest) + +ThrottleState = _reflection.GeneratedProtocolMessageType( + "ThrottleState", + (_message.Message,), + dict( + DESCRIPTOR=_THROTTLESTATE, + __module__="google.cloud.bigquery.storage_v1beta2.proto.storage_pb2", + __doc__="""Information on if the current connection is being + throttled. + + + Attributes: + throttle_percent: + How much this connection is being throttled. Zero means no + throttling, 100 means fully throttled. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ThrottleState) + ), +) +_sym_db.RegisterMessage(ThrottleState) + +StreamStats = _reflection.GeneratedProtocolMessageType( + "StreamStats", + (_message.Message,), + dict( + Progress=_reflection.GeneratedProtocolMessageType( + "Progress", + (_message.Message,), + dict( + DESCRIPTOR=_STREAMSTATS_PROGRESS, + __module__="google.cloud.bigquery.storage_v1beta2.proto.storage_pb2", + __doc__="""Protocol buffer. + + Attributes: + at_response_start: + The fraction of rows assigned to the stream that have been + processed by the server so far, not including the rows in the + current response message. This value, along with + ``at_response_end``, can be used to interpolate the progress + made as the rows in the message are being processed using the + following formula: ``at_response_start + (at_response_end - + at_response_start) * rows_processed_from_response / + rows_in_response``. Note that if a filter is provided, the + ``at_response_end`` value of the previous response may not + necessarily be equal to the ``at_response_start`` value of the + current response. + at_response_end: + Similar to ``at_response_start``, except that this value + includes the rows in the current response. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.StreamStats.Progress) + ), + ), + DESCRIPTOR=_STREAMSTATS, + __module__="google.cloud.bigquery.storage_v1beta2.proto.storage_pb2", + __doc__="""Estimated stream statistics for a given Stream. + + + Attributes: + progress: + Represents the progress of the current stream. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.StreamStats) + ), +) +_sym_db.RegisterMessage(StreamStats) +_sym_db.RegisterMessage(StreamStats.Progress) + +ReadRowsResponse = _reflection.GeneratedProtocolMessageType( + "ReadRowsResponse", + (_message.Message,), + dict( + DESCRIPTOR=_READROWSRESPONSE, + __module__="google.cloud.bigquery.storage_v1beta2.proto.storage_pb2", + __doc__="""Response from calling ``ReadRows`` may include row data, + progress and throttling information. + + + Attributes: + rows: + Row data is returned in format specified during session + creation. + avro_rows: + Serialized row data in AVRO format. + arrow_record_batch: + Serialized row data in Arrow RecordBatch format. + row_count: + Number of serialized rows in the rows block. + stats: + Statistics for the stream. + throttle_state: + Throttling state. If unset, the latest response still + describes the current throttling status. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ReadRowsResponse) + ), +) +_sym_db.RegisterMessage(ReadRowsResponse) + +SplitReadStreamRequest = _reflection.GeneratedProtocolMessageType( + "SplitReadStreamRequest", + (_message.Message,), + dict( + DESCRIPTOR=_SPLITREADSTREAMREQUEST, + __module__="google.cloud.bigquery.storage_v1beta2.proto.storage_pb2", + __doc__="""Request message for ``SplitReadStream``. + + + Attributes: + name: + Required. Name of the stream to split. + fraction: + A value in the range (0.0, 1.0) that specifies the fractional + point at which the original stream should be split. The actual + split point is evaluated on pre-filtered rows, so if a filter + is provided, then there is no guarantee that the division of + the rows between the new child streams will be proportional to + this fractional value. Additionally, because the server-side + unit for assigning data is collections of rows, this fraction + will always map to a data storage boundary on the server side. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.SplitReadStreamRequest) + ), +) +_sym_db.RegisterMessage(SplitReadStreamRequest) + +SplitReadStreamResponse = _reflection.GeneratedProtocolMessageType( + "SplitReadStreamResponse", + (_message.Message,), + dict( + DESCRIPTOR=_SPLITREADSTREAMRESPONSE, + __module__="google.cloud.bigquery.storage_v1beta2.proto.storage_pb2", + __doc__="""Response message for ``SplitReadStream``. + + + Attributes: + primary_stream: + Primary stream, which contains the beginning portion of + \|original\_stream\|. An empty value indicates that the + original stream can no longer be split. + remainder_stream: + Remainder stream, which contains the tail of + \|original\_stream\|. An empty value indicates that the + original stream can no longer be split. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.SplitReadStreamResponse) + ), +) +_sym_db.RegisterMessage(SplitReadStreamResponse) + + +DESCRIPTOR._options = None +_CREATEREADSESSIONREQUEST.fields_by_name["parent"]._options = None +_CREATEREADSESSIONREQUEST.fields_by_name["read_session"]._options = None +_READROWSREQUEST.fields_by_name["read_stream"]._options = None +_SPLITREADSTREAMREQUEST.fields_by_name["name"]._options = None + +_BIGQUERYREAD = _descriptor.ServiceDescriptor( + name="BigQueryRead", + full_name="google.cloud.bigquery.storage.v1beta2.BigQueryRead", + file=DESCRIPTOR, + index=0, + serialized_options=_b( + "\312A\036bigquerystorage.googleapis.com\322A\211\001https://www.googleapis.com/auth/bigquery,https://www.googleapis.com/auth/bigquery.readonly,https://www.googleapis.com/auth/cloud-platform" + ), + serialized_start=1538, + serialized_end=2421, + methods=[ + _descriptor.MethodDescriptor( + name="CreateReadSession", + full_name="google.cloud.bigquery.storage.v1beta2.BigQueryRead.CreateReadSession", + index=0, + containing_service=None, + input_type=_CREATEREADSESSIONREQUEST, + output_type=google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_stream__pb2._READSESSION, + serialized_options=_b( + '\202\323\344\223\002A" 5" + // "date_field = CAST('2014-9-27' as DATE)" + // "nullable_field is not NULL" + // "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))" + // "numeric_field BETWEEN 1.0 AND 5.0" + string row_restriction = 2; + + // Optional. Options specific to the Apache Arrow output format. + ArrowSerializationOptions arrow_serialization_options = 3 [(google.api.field_behavior) = OPTIONAL]; + } + + // Output only. Unique identifier for the session, in the form + // `projects/{project_id}/locations/{location}/sessions/{session_id}`. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Time at which the session becomes invalid. After this time, subsequent + // requests to read this Session will return errors. The expire_time is + // automatically assigned and currently cannot be specified or updated. + google.protobuf.Timestamp expire_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Immutable. Data format of the output data. + DataFormat data_format = 3 [(google.api.field_behavior) = IMMUTABLE]; + + // The schema for the read. If read_options.selected_fields is set, the + // schema may be different from the table schema as it will only contain + // the selected fields. + oneof schema { + // Output only. Avro schema. + AvroSchema avro_schema = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Arrow schema. + ArrowSchema arrow_schema = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + } + + // Immutable. Table that this ReadSession is reading from, in the form + // `projects/{project_id}/datasets/{dataset_id}/tables/{table_id} + string table = 6 [ + (google.api.field_behavior) = IMMUTABLE, + (google.api.resource_reference) = { + type: "bigquery.googleapis.com/Table" + } + ]; + + // Optional. Any modifiers which are applied when reading from the specified table. + TableModifiers table_modifiers = 7 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Read options for this session (e.g. column selection, filters). + TableReadOptions read_options = 8 [(google.api.field_behavior) = OPTIONAL]; + + // Output only. A list of streams created with the session. + // + // At least one stream is created with the session. In the future, larger + // request_stream_count values *may* result in this list being unpopulated, + // in that case, the user will need to use a List method to get the streams + // instead, which is not yet available. + repeated ReadStream streams = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; +} + +// Information about a single stream that gets data out of the storage system. +// Most of the information about `ReadStream` instances is aggregated, making +// `ReadStream` lightweight. +message ReadStream { + option (google.api.resource) = { + type: "bigquerystorage.googleapis.com/ReadStream" + pattern: "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}" + }; + + // Output only. Name of the stream, in the form + // `projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}`. + string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; +} diff --git a/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2.py b/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2.py new file mode 100644 index 00000000..1a597691 --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2.py @@ -0,0 +1,615 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: google/cloud/bigquery/storage_v1beta2/proto/stream.proto + +import sys + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 +from google.api import resource_pb2 as google_dot_api_dot_resource__pb2 +from google.cloud.bigquery.storage_v1beta2.proto import ( + arrow_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_arrow__pb2, +) +from google.cloud.bigquery.storage_v1beta2.proto import ( + avro_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_avro__pb2, +) +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="google/cloud/bigquery/storage_v1beta2/proto/stream.proto", + package="google.cloud.bigquery.storage.v1beta2", + syntax="proto3", + serialized_options=_b( + "\n)com.google.cloud.bigquery.storage.v1beta2B\013StreamProtoP\001ZLgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storage\352AU\n\035bigquery.googleapis.com/Table\0224projects/{project}/datasets/{dataset}/tables/{table}" + ), + serialized_pb=_b( + '\n8google/cloud/bigquery/storage_v1beta2/proto/stream.proto\x12%google.cloud.bigquery.storage.v1beta2\x1a\x1fgoogle/api/field_behavior.proto\x1a\x19google/api/resource.proto\x1a\x37google/cloud/bigquery/storage_v1beta2/proto/arrow.proto\x1a\x36google/cloud/bigquery/storage_v1beta2/proto/avro.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\xf2\x07\n\x0bReadSession\x12\x11\n\x04name\x18\x01 \x01(\tB\x03\xe0\x41\x03\x12\x34\n\x0b\x65xpire_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03\x12K\n\x0b\x64\x61ta_format\x18\x03 \x01(\x0e\x32\x31.google.cloud.bigquery.storage.v1beta2.DataFormatB\x03\xe0\x41\x05\x12M\n\x0b\x61vro_schema\x18\x04 \x01(\x0b\x32\x31.google.cloud.bigquery.storage.v1beta2.AvroSchemaB\x03\xe0\x41\x03H\x00\x12O\n\x0c\x61rrow_schema\x18\x05 \x01(\x0b\x32\x32.google.cloud.bigquery.storage.v1beta2.ArrowSchemaB\x03\xe0\x41\x03H\x00\x12\x34\n\x05table\x18\x06 \x01(\tB%\xe0\x41\x05\xfa\x41\x1f\n\x1d\x62igquery.googleapis.com/Table\x12_\n\x0ftable_modifiers\x18\x07 \x01(\x0b\x32\x41.google.cloud.bigquery.storage.v1beta2.ReadSession.TableModifiersB\x03\xe0\x41\x01\x12^\n\x0cread_options\x18\x08 \x01(\x0b\x32\x43.google.cloud.bigquery.storage.v1beta2.ReadSession.TableReadOptionsB\x03\xe0\x41\x01\x12G\n\x07streams\x18\n \x03(\x0b\x32\x31.google.cloud.bigquery.storage.v1beta2.ReadStreamB\x03\xe0\x41\x03\x1a\x43\n\x0eTableModifiers\x12\x31\n\rsnapshot_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\xb0\x01\n\x10TableReadOptions\x12\x17\n\x0fselected_fields\x18\x01 \x03(\t\x12\x17\n\x0frow_restriction\x18\x02 \x01(\t\x12j\n\x1b\x61rrow_serialization_options\x18\x03 \x01(\x0b\x32@.google.cloud.bigquery.storage.v1beta2.ArrowSerializationOptionsB\x03\xe0\x41\x01:k\xea\x41h\n*bigquerystorage.googleapis.com/ReadSession\x12:projects/{project}/locations/{location}/sessions/{session}B\x08\n\x06schema"\x9c\x01\n\nReadStream\x12\x11\n\x04name\x18\x01 \x01(\tB\x03\xe0\x41\x03:{\xea\x41x\n)bigquerystorage.googleapis.com/ReadStream\x12Kprojects/{project}/locations/{location}/sessions/{session}/streams/{stream}*>\n\nDataFormat\x12\x1b\n\x17\x44\x41TA_FORMAT_UNSPECIFIED\x10\x00\x12\x08\n\x04\x41VRO\x10\x01\x12\t\n\x05\x41RROW\x10\x02\x42\xe0\x01\n)com.google.cloud.bigquery.storage.v1beta2B\x0bStreamProtoP\x01ZLgoogle.golang.org/genproto/googleapis/cloud/bigquery/storage/v1beta2;storage\xea\x41U\n\x1d\x62igquery.googleapis.com/Table\x12\x34projects/{project}/datasets/{dataset}/tables/{table}b\x06proto3' + ), + dependencies=[ + google_dot_api_dot_field__behavior__pb2.DESCRIPTOR, + google_dot_api_dot_resource__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_arrow__pb2.DESCRIPTOR, + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_avro__pb2.DESCRIPTOR, + google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR, + ], +) + +_DATAFORMAT = _descriptor.EnumDescriptor( + name="DataFormat", + full_name="google.cloud.bigquery.storage.v1beta2.DataFormat", + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name="DATA_FORMAT_UNSPECIFIED", + index=0, + number=0, + serialized_options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name="AVRO", index=1, number=1, serialized_options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ARROW", index=2, number=2, serialized_options=None, type=None + ), + ], + containing_type=None, + serialized_options=None, + serialized_start=1477, + serialized_end=1539, +) +_sym_db.RegisterEnumDescriptor(_DATAFORMAT) + +DataFormat = enum_type_wrapper.EnumTypeWrapper(_DATAFORMAT) +DATA_FORMAT_UNSPECIFIED = 0 +AVRO = 1 +ARROW = 2 + + +_READSESSION_TABLEMODIFIERS = _descriptor.Descriptor( + name="TableModifiers", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.TableModifiers", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="snapshot_time", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.TableModifiers.snapshot_time", + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=951, + serialized_end=1018, +) + +_READSESSION_TABLEREADOPTIONS = _descriptor.Descriptor( + name="TableReadOptions", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.TableReadOptions", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="selected_fields", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.TableReadOptions.selected_fields", + index=0, + number=1, + type=9, + cpp_type=9, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="row_restriction", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.TableReadOptions.row_restriction", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=None, + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="arrow_serialization_options", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.TableReadOptions.arrow_serialization_options", + index=2, + number=3, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\001"), + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=1021, + serialized_end=1197, +) + +_READSESSION = _descriptor.Descriptor( + name="ReadSession", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="name", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.name", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="expire_time", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.expire_time", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="data_format", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.data_format", + index=2, + number=3, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\005"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="avro_schema", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.avro_schema", + index=3, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="arrow_schema", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.arrow_schema", + index=4, + number=5, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="table", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.table", + index=5, + number=6, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b( + "\340A\005\372A\037\n\035bigquery.googleapis.com/Table" + ), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="table_modifiers", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.table_modifiers", + index=6, + number=7, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\001"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="read_options", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.read_options", + index=7, + number=8, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\001"), + file=DESCRIPTOR, + ), + _descriptor.FieldDescriptor( + name="streams", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.streams", + index=8, + number=10, + type=11, + cpp_type=10, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ), + ], + extensions=[], + nested_types=[_READSESSION_TABLEMODIFIERS, _READSESSION_TABLEREADOPTIONS], + enum_types=[], + serialized_options=_b( + "\352Ah\n*bigquerystorage.googleapis.com/ReadSession\022:projects/{project}/locations/{location}/sessions/{session}" + ), + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name="schema", + full_name="google.cloud.bigquery.storage.v1beta2.ReadSession.schema", + index=0, + containing_type=None, + fields=[], + ) + ], + serialized_start=306, + serialized_end=1316, +) + + +_READSTREAM = _descriptor.Descriptor( + name="ReadStream", + full_name="google.cloud.bigquery.storage.v1beta2.ReadStream", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="name", + full_name="google.cloud.bigquery.storage.v1beta2.ReadStream.name", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + serialized_options=_b("\340A\003"), + file=DESCRIPTOR, + ) + ], + extensions=[], + nested_types=[], + enum_types=[], + serialized_options=_b( + "\352Ax\n)bigquerystorage.googleapis.com/ReadStream\022Kprojects/{project}/locations/{location}/sessions/{session}/streams/{stream}" + ), + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=1319, + serialized_end=1475, +) + +_READSESSION_TABLEMODIFIERS.fields_by_name[ + "snapshot_time" +].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +_READSESSION_TABLEMODIFIERS.containing_type = _READSESSION +_READSESSION_TABLEREADOPTIONS.fields_by_name[ + "arrow_serialization_options" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_arrow__pb2._ARROWSERIALIZATIONOPTIONS +) +_READSESSION_TABLEREADOPTIONS.containing_type = _READSESSION +_READSESSION.fields_by_name[ + "expire_time" +].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +_READSESSION.fields_by_name["data_format"].enum_type = _DATAFORMAT +_READSESSION.fields_by_name[ + "avro_schema" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_avro__pb2._AVROSCHEMA +) +_READSESSION.fields_by_name[ + "arrow_schema" +].message_type = ( + google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_arrow__pb2._ARROWSCHEMA +) +_READSESSION.fields_by_name[ + "table_modifiers" +].message_type = _READSESSION_TABLEMODIFIERS +_READSESSION.fields_by_name["read_options"].message_type = _READSESSION_TABLEREADOPTIONS +_READSESSION.fields_by_name["streams"].message_type = _READSTREAM +_READSESSION.oneofs_by_name["schema"].fields.append( + _READSESSION.fields_by_name["avro_schema"] +) +_READSESSION.fields_by_name[ + "avro_schema" +].containing_oneof = _READSESSION.oneofs_by_name["schema"] +_READSESSION.oneofs_by_name["schema"].fields.append( + _READSESSION.fields_by_name["arrow_schema"] +) +_READSESSION.fields_by_name[ + "arrow_schema" +].containing_oneof = _READSESSION.oneofs_by_name["schema"] +DESCRIPTOR.message_types_by_name["ReadSession"] = _READSESSION +DESCRIPTOR.message_types_by_name["ReadStream"] = _READSTREAM +DESCRIPTOR.enum_types_by_name["DataFormat"] = _DATAFORMAT +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +ReadSession = _reflection.GeneratedProtocolMessageType( + "ReadSession", + (_message.Message,), + dict( + TableModifiers=_reflection.GeneratedProtocolMessageType( + "TableModifiers", + (_message.Message,), + dict( + DESCRIPTOR=_READSESSION_TABLEMODIFIERS, + __module__="google.cloud.bigquery.storage_v1beta2.proto.stream_pb2", + __doc__="""Additional attributes when reading a table. + + + Attributes: + snapshot_time: + The snapshot time of the table. If not set, interpreted as + now. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ReadSession.TableModifiers) + ), + ), + TableReadOptions=_reflection.GeneratedProtocolMessageType( + "TableReadOptions", + (_message.Message,), + dict( + DESCRIPTOR=_READSESSION_TABLEREADOPTIONS, + __module__="google.cloud.bigquery.storage_v1beta2.proto.stream_pb2", + __doc__="""Options dictating how we read a table. + + + Attributes: + selected_fields: + Names of the fields in the table that should be read. If + empty, all fields will be read. If the specified field is a + nested field, all the sub-fields in the field will be + selected. The output field order is unrelated to the order of + fields in selected\_fields. + row_restriction: + SQL text filtering statement, similar to a WHERE clause in a + query. Currently, only a single predicate that is a comparison + between a column and a constant value is supported. Aggregates + are not supported. Examples: "int\_field > 5" "date\_field = + CAST('2014-9-27' as DATE)" "nullable\_field is not NULL" + "st\_equals(geo\_field, st\_geofromtext("POINT(2, 2)"))" + "numeric\_field BETWEEN 1.0 AND 5.0" + arrow_serialization_options: + Optional. Options specific to the Apache Arrow output format. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ReadSession.TableReadOptions) + ), + ), + DESCRIPTOR=_READSESSION, + __module__="google.cloud.bigquery.storage_v1beta2.proto.stream_pb2", + __doc__="""Information about the ReadSession. + + + Attributes: + name: + Output only. Unique identifier for the session, in the form `` + projects/{project_id}/locations/{location}/sessions/{session_i + d}``. + expire_time: + Output only. Time at which the session becomes invalid. After + this time, subsequent requests to read this Session will + return errors. The expire\_time is automatically assigned and + currently cannot be specified or updated. + data_format: + Immutable. Data format of the output data. + schema: + The schema for the read. If read\_options.selected\_fields is + set, the schema may be different from the table schema as it + will only contain the selected fields. + avro_schema: + Output only. Avro schema. + arrow_schema: + Output only. Arrow schema. + table: + Immutable. Table that this ReadSession is reading from, in the + form \`projects/{project\_id}/datasets/{dataset\_id}/tables/{t + able\_id} + table_modifiers: + Optional. Any modifiers which are applied when reading from + the specified table. + read_options: + Optional. Read options for this session (e.g. column + selection, filters). + streams: + Output only. A list of streams created with the session. At + least one stream is created with the session. In the future, + larger request\_stream\_count values *may* result in this list + being unpopulated, in that case, the user will need to use a + List method to get the streams instead, which is not yet + available. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ReadSession) + ), +) +_sym_db.RegisterMessage(ReadSession) +_sym_db.RegisterMessage(ReadSession.TableModifiers) +_sym_db.RegisterMessage(ReadSession.TableReadOptions) + +ReadStream = _reflection.GeneratedProtocolMessageType( + "ReadStream", + (_message.Message,), + dict( + DESCRIPTOR=_READSTREAM, + __module__="google.cloud.bigquery.storage_v1beta2.proto.stream_pb2", + __doc__="""Information about a single stream that gets data out of + the storage system. Most of the information about ``ReadStream`` + instances is aggregated, making ``ReadStream`` lightweight. + + + Attributes: + name: + Output only. Name of the stream, in the form ``projects/{proje + ct_id}/locations/{location}/sessions/{session_id}/streams/{str + eam_id}``. + """, + # @@protoc_insertion_point(class_scope:google.cloud.bigquery.storage.v1beta2.ReadStream) + ), +) +_sym_db.RegisterMessage(ReadStream) + + +DESCRIPTOR._options = None +_READSESSION_TABLEREADOPTIONS.fields_by_name[ + "arrow_serialization_options" +]._options = None +_READSESSION.fields_by_name["name"]._options = None +_READSESSION.fields_by_name["expire_time"]._options = None +_READSESSION.fields_by_name["data_format"]._options = None +_READSESSION.fields_by_name["avro_schema"]._options = None +_READSESSION.fields_by_name["arrow_schema"]._options = None +_READSESSION.fields_by_name["table"]._options = None +_READSESSION.fields_by_name["table_modifiers"]._options = None +_READSESSION.fields_by_name["read_options"]._options = None +_READSESSION.fields_by_name["streams"]._options = None +_READSESSION._options = None +_READSTREAM.fields_by_name["name"]._options = None +_READSTREAM._options = None +# @@protoc_insertion_point(module_scope) diff --git a/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2_grpc.py b/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2_grpc.py new file mode 100644 index 00000000..07cb78fe --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2_grpc.py @@ -0,0 +1,2 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc diff --git a/google/cloud/bigquery_storage_v1beta2/types.py b/google/cloud/bigquery_storage_v1beta2/types.py new file mode 100644 index 00000000..da6417f2 --- /dev/null +++ b/google/cloud/bigquery_storage_v1beta2/types.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import absolute_import +import sys + +from google.api_core.protobuf_helpers import get_messages + +from google.cloud.bigquery_storage_v1beta2.proto import arrow_pb2 +from google.cloud.bigquery_storage_v1beta2.proto import avro_pb2 +from google.cloud.bigquery_storage_v1beta2.proto import storage_pb2 +from google.cloud.bigquery_storage_v1beta2.proto import stream_pb2 +from google.protobuf import timestamp_pb2 + + +_shared_modules = [timestamp_pb2] + +_local_modules = [arrow_pb2, avro_pb2, storage_pb2, stream_pb2] + +names = [] + +for module in _shared_modules: # pragma: NO COVER + for name, message in get_messages(module).items(): + setattr(sys.modules[__name__], name, message) + names.append(name) +for module in _local_modules: + for name, message in get_messages(module).items(): + message.__module__ = "google.cloud.bigquery_storage_v1beta2.types" + setattr(sys.modules[__name__], name, message) + names.append(name) + + +__all__ = tuple(sorted(names)) diff --git a/synth.metadata b/synth.metadata index 39db5a30..417239cb 100644 --- a/synth.metadata +++ b/synth.metadata @@ -1,5 +1,5 @@ { - "updateTime": "2020-02-12T18:06:09.446329Z", + "updateTime": "2020-02-19T19:57:32.518082Z", "sources": [ { "generator": { @@ -12,8 +12,8 @@ "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "efd36705972cfcd7d00ab4c6dfa1135bafacd4ae", - "internalRef": "294664231" + "sha": "a9639a0a9854fd6e1be08bba1ac3897f4f16cb2f", + "internalRef": "295983266" } }, { @@ -34,6 +34,26 @@ "generator": "gapic", "config": "google/cloud/bigquery/storage/artman_bigquerystorage_v1beta1.yaml" } + }, + { + "client": { + "source": "googleapis", + "apiName": "bigquery_storage", + "apiVersion": "v1beta2", + "language": "python", + "generator": "gapic", + "config": "google/cloud/bigquery/storage/artman_bigquerystorage_v1beta2.yaml" + } + }, + { + "client": { + "source": "googleapis", + "apiName": "bigquery_storage", + "apiVersion": "v1", + "language": "python", + "generator": "gapic", + "config": "google/cloud/bigquery/storage/artman_bigquerystorage_v1.yaml" + } } ] } \ No newline at end of file diff --git a/synth.py b/synth.py index cd146dc9..bd828ca0 100644 --- a/synth.py +++ b/synth.py @@ -19,8 +19,7 @@ gapic = gcp.GAPICGenerator() common = gcp.CommonTemplates() -versions = ["v1beta1"] - +versions = ["v1beta1", "v1beta2", "v1"] for version in versions: @@ -45,6 +44,36 @@ ], ) + # We need to parameterize aspects of the client as it varies in different versions. + # + # In the future once the read and write client are colocated in the same version, + # we'll need to loop through through multiple clients. Perhaps by the time that + # happens we'll be on a generator that needs less post-generation modifications. + + clientinfo = { + "file": "big_query_storage_client.py", + "type": "storage", + "name": "BigQueryStorageClient", + "badpkg": "google-cloud-bigquerystorage", + "goodpkg": "google-cloud-bigquery-storage", + } + if version in ["v1beta2","v1"]: + clientinfo = { + "file": "big_query_read_client.py", + "type": "read", + "name": "BigQueryReadClient", + "badpkg": "google-cloud-bigquerystorage", + "goodpkg": "google-cloud-bigquery-storage", + } + if version in ["v1alpha2"]: + clientinfo = { + "file": "big_query_write_client.py", + "type": "write", + "name": "BigQueryWriteClient", + "badpkg": "google-cloud-bigquerystorage", + "goodpkg": "google-cloud-bigquery-storage", + } + s.replace( [ f"google/cloud/bigquery_storage_{version}/proto/storage_pb2.py", @@ -54,46 +83,47 @@ f"from google.cloud.bigquery_storage_{version}.proto", ) + # This is used to populate _GAPIC_LIBRARY_VERSION in the client. s.replace( - f"google/cloud/bigquery_storage_{version}/gapic/" "big_query_storage_client.py", - "google-cloud-bigquerystorage", - "google-cloud-bigquery-storage", + f"google/cloud/bigquery_storage_{version}/gapic/{clientinfo['file']}", + clientinfo['badpkg'], + clientinfo['goodpkg'] ) s.replace( - f"google/cloud/bigquery_storage_{version}/gapic/" "big_query_storage_client.py", + f"google/cloud/bigquery_storage_{version}/gapic/{clientinfo['file']}", "import google.api_core.gapic_v1.method\n", "\g<0>import google.api_core.path_template\n", ) s.replace( - [f"tests/unit/gapic/{version}/test_big_query_storage_client_{version}.py"], + [f"tests/unit/gapic/{version}/test_big_query_{clientinfo['type']}_client_{version}.py"], f"from google.cloud import bigquery_storage_{version}", f"from google.cloud.bigquery_storage_{version}.gapic import big_query_storage_client # noqa", ) s.replace( - [f"tests/unit/gapic/{version}/test_big_query_storage_client_{version}.py"], - "bigquery_storage_v1beta1.BigQueryStorageClient", - "big_query_storage_client.BigQueryStorageClient", + [f"tests/unit/gapic/{version}/test_big_query_{clientinfo['type']}_client_{version}.py"], + f"bigquery_storage_{version}.{clientinfo['name']}", + f"big_query_storage_client.{clientinfo['name']}", ) # START: Ignore lint and coverage s.replace( - [f"google/cloud/bigquery_storage_{version}/gapic/big_query_storage_client.py"], + [f"google/cloud/bigquery_storage_{version}/gapic/big_query_{clientinfo['type']}_client.py"], "if transport:", "if transport: # pragma: no cover", ) s.replace( - [f"google/cloud/bigquery_storage_{version}/gapic/big_query_storage_client.py"], + [f"google/cloud/bigquery_storage_{version}/gapic/big_query_{clientinfo['type']}_client.py"], r"metadata.append\(routing_metadata\)", "metadata.append(routing_metadata) # pragma: no cover", ) s.replace( [ - f"google/cloud/bigquery_storage_{version}/gapic/transports/big_query_storage_grpc_transport.py" + f"google/cloud/bigquery_storage_{version}/gapic/transports/big_query_{clientinfo['type']}_grpc_transport.py" ], "if channel is not None and credentials is not None:", "if channel is not None and credentials is not None: # pragma: no cover", @@ -101,7 +131,7 @@ s.replace( [ - f"google/cloud/bigquery_storage_{version}/gapic/transports/big_query_storage_grpc_transport.py" + f"google/cloud/bigquery_storage_{version}/gapic/transports/big_query_{clientinfo['type']}_grpc_transport.py" ], "if channel is None:", "if channel is None: # pragma: no cover", @@ -109,7 +139,7 @@ s.replace( [ - f"google/cloud/bigquery_storage_{version}/gapic/transports/big_query_storage_grpc_transport.py" + f"google/cloud/bigquery_storage_{version}/gapic/transports/big_query_{clientinfo['type']}_grpc_transport.py" ], r"google.api_core.grpc_helpers.create_channel\(", "google.api_core.grpc_helpers.create_channel( # pragma: no cover", diff --git a/tests/unit/gapic/v1/test_big_query_read_client_v1.py b/tests/unit/gapic/v1/test_big_query_read_client_v1.py new file mode 100644 index 00000000..5f4c1f7d --- /dev/null +++ b/tests/unit/gapic/v1/test_big_query_read_client_v1.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests.""" + +import mock +import pytest + +from google.cloud.bigquery_storage_v1.gapic import big_query_storage_client # noqa +from google.cloud.bigquery_storage_v1.proto import storage_pb2 +from google.cloud.bigquery_storage_v1.proto import stream_pb2 + + +class MultiCallableStub(object): + """Stub for the grpc.UnaryUnaryMultiCallable interface.""" + + def __init__(self, method, channel_stub): + self.method = method + self.channel_stub = channel_stub + + def __call__(self, request, timeout=None, metadata=None, credentials=None): + self.channel_stub.requests.append((self.method, request)) + + response = None + if self.channel_stub.responses: + response = self.channel_stub.responses.pop() + + if isinstance(response, Exception): + raise response + + if response: + return response + + +class ChannelStub(object): + """Stub for the grpc.Channel interface.""" + + def __init__(self, responses=[]): + self.responses = responses + self.requests = [] + + def unary_unary(self, method, request_serializer=None, response_deserializer=None): + return MultiCallableStub(method, self) + + def unary_stream(self, method, request_serializer=None, response_deserializer=None): + return MultiCallableStub(method, self) + + +class CustomException(Exception): + pass + + +class TestBigQueryReadClient(object): + def test_create_read_session(self): + # Setup Expected Response + name = "name3373707" + table = "table110115790" + expected_response = {"name": name, "table": table} + expected_response = stream_pb2.ReadSession(**expected_response) + + # Mock the API response + channel = ChannelStub(responses=[expected_response]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + response = client.create_read_session() + assert expected_response == response + + assert len(channel.requests) == 1 + expected_request = storage_pb2.CreateReadSessionRequest() + actual_request = channel.requests[0][1] + assert expected_request == actual_request + + def test_create_read_session_exception(self): + # Mock the API response + channel = ChannelStub(responses=[CustomException()]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + with pytest.raises(CustomException): + client.create_read_session() + + def test_read_rows(self): + # Setup Expected Response + row_count = 1340416618 + expected_response = {"row_count": row_count} + expected_response = storage_pb2.ReadRowsResponse(**expected_response) + + # Mock the API response + channel = ChannelStub(responses=[iter([expected_response])]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + response = client.read_rows() + resources = list(response) + assert len(resources) == 1 + assert expected_response == resources[0] + + assert len(channel.requests) == 1 + expected_request = storage_pb2.ReadRowsRequest() + actual_request = channel.requests[0][1] + assert expected_request == actual_request + + def test_read_rows_exception(self): + # Mock the API response + channel = ChannelStub(responses=[CustomException()]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + with pytest.raises(CustomException): + client.read_rows() + + def test_split_read_stream(self): + # Setup Expected Response + expected_response = {} + expected_response = storage_pb2.SplitReadStreamResponse(**expected_response) + + # Mock the API response + channel = ChannelStub(responses=[expected_response]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + response = client.split_read_stream() + assert expected_response == response + + assert len(channel.requests) == 1 + expected_request = storage_pb2.SplitReadStreamRequest() + actual_request = channel.requests[0][1] + assert expected_request == actual_request + + def test_split_read_stream_exception(self): + # Mock the API response + channel = ChannelStub(responses=[CustomException()]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + with pytest.raises(CustomException): + client.split_read_stream() diff --git a/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py b/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py new file mode 100644 index 00000000..82edfeb2 --- /dev/null +++ b/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests.""" + +import mock +import pytest + +from google.cloud.bigquery_storage_v1beta2.gapic import big_query_storage_client # noqa +from google.cloud.bigquery_storage_v1beta2.proto import storage_pb2 +from google.cloud.bigquery_storage_v1beta2.proto import stream_pb2 + + +class MultiCallableStub(object): + """Stub for the grpc.UnaryUnaryMultiCallable interface.""" + + def __init__(self, method, channel_stub): + self.method = method + self.channel_stub = channel_stub + + def __call__(self, request, timeout=None, metadata=None, credentials=None): + self.channel_stub.requests.append((self.method, request)) + + response = None + if self.channel_stub.responses: + response = self.channel_stub.responses.pop() + + if isinstance(response, Exception): + raise response + + if response: + return response + + +class ChannelStub(object): + """Stub for the grpc.Channel interface.""" + + def __init__(self, responses=[]): + self.responses = responses + self.requests = [] + + def unary_unary(self, method, request_serializer=None, response_deserializer=None): + return MultiCallableStub(method, self) + + def unary_stream(self, method, request_serializer=None, response_deserializer=None): + return MultiCallableStub(method, self) + + +class CustomException(Exception): + pass + + +class TestBigQueryReadClient(object): + def test_create_read_session(self): + # Setup Expected Response + name = "name3373707" + table = "table110115790" + expected_response = {"name": name, "table": table} + expected_response = stream_pb2.ReadSession(**expected_response) + + # Mock the API response + channel = ChannelStub(responses=[expected_response]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + response = client.create_read_session() + assert expected_response == response + + assert len(channel.requests) == 1 + expected_request = storage_pb2.CreateReadSessionRequest() + actual_request = channel.requests[0][1] + assert expected_request == actual_request + + def test_create_read_session_exception(self): + # Mock the API response + channel = ChannelStub(responses=[CustomException()]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + with pytest.raises(CustomException): + client.create_read_session() + + def test_read_rows(self): + # Setup Expected Response + row_count = 1340416618 + expected_response = {"row_count": row_count} + expected_response = storage_pb2.ReadRowsResponse(**expected_response) + + # Mock the API response + channel = ChannelStub(responses=[iter([expected_response])]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + response = client.read_rows() + resources = list(response) + assert len(resources) == 1 + assert expected_response == resources[0] + + assert len(channel.requests) == 1 + expected_request = storage_pb2.ReadRowsRequest() + actual_request = channel.requests[0][1] + assert expected_request == actual_request + + def test_read_rows_exception(self): + # Mock the API response + channel = ChannelStub(responses=[CustomException()]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + with pytest.raises(CustomException): + client.read_rows() + + def test_split_read_stream(self): + # Setup Expected Response + expected_response = {} + expected_response = storage_pb2.SplitReadStreamResponse(**expected_response) + + # Mock the API response + channel = ChannelStub(responses=[expected_response]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + response = client.split_read_stream() + assert expected_response == response + + assert len(channel.requests) == 1 + expected_request = storage_pb2.SplitReadStreamRequest() + actual_request = channel.requests[0][1] + assert expected_request == actual_request + + def test_split_read_stream_exception(self): + # Mock the API response + channel = ChannelStub(responses=[CustomException()]) + patch = mock.patch("google.api_core.grpc_helpers.create_channel") + with patch as create_channel: + create_channel.return_value = channel + client = big_query_storage_client.BigQueryReadClient() + + with pytest.raises(CustomException): + client.split_read_stream() From 9151c527229cccb7c5082437cde982bc5e10eb31 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 19 Feb 2020 20:24:55 +0000 Subject: [PATCH 2/7] add another variable to replacements --- google/cloud/bigquery_storage_v1/proto/stream_pb2.py | 6 +++--- synth.metadata | 2 +- synth.py | 2 +- tests/unit/gapic/v1/test_big_query_read_client_v1.py | 2 +- .../gapic/v1beta2/test_big_query_read_client_v1beta2.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/google/cloud/bigquery_storage_v1/proto/stream_pb2.py b/google/cloud/bigquery_storage_v1/proto/stream_pb2.py index 46f1177b..0d52a8de 100644 --- a/google/cloud/bigquery_storage_v1/proto/stream_pb2.py +++ b/google/cloud/bigquery_storage_v1/proto/stream_pb2.py @@ -554,9 +554,9 @@ dict( DESCRIPTOR=_READSTREAM, __module__="google.cloud.bigquery.storage_v1.proto.stream_pb2", - __doc__="""Information about a single stream that gets data out of - the storage system. Most of the information about ``ReadStream`` - instances is aggregated, making ``ReadStream`` lightweight. + __doc__="""Information about a single stream that gets data out of the storage + system. Most of the information about ``ReadStream`` instances is + aggregated, making ``ReadStream`` lightweight. Attributes: diff --git a/synth.metadata b/synth.metadata index 417239cb..fedd0785 100644 --- a/synth.metadata +++ b/synth.metadata @@ -1,5 +1,5 @@ { - "updateTime": "2020-02-19T19:57:32.518082Z", + "updateTime": "2020-02-19T20:23:03.706005Z", "sources": [ { "generator": { diff --git a/synth.py b/synth.py index bd828ca0..f83db596 100644 --- a/synth.py +++ b/synth.py @@ -99,7 +99,7 @@ s.replace( [f"tests/unit/gapic/{version}/test_big_query_{clientinfo['type']}_client_{version}.py"], f"from google.cloud import bigquery_storage_{version}", - f"from google.cloud.bigquery_storage_{version}.gapic import big_query_storage_client # noqa", + f"from google.cloud.bigquery_storage_{version}.gapic import big_query_{clientinfo['type']}_client # noqa", ) s.replace( diff --git a/tests/unit/gapic/v1/test_big_query_read_client_v1.py b/tests/unit/gapic/v1/test_big_query_read_client_v1.py index 5f4c1f7d..93d775ad 100644 --- a/tests/unit/gapic/v1/test_big_query_read_client_v1.py +++ b/tests/unit/gapic/v1/test_big_query_read_client_v1.py @@ -19,7 +19,7 @@ import mock import pytest -from google.cloud.bigquery_storage_v1.gapic import big_query_storage_client # noqa +from google.cloud.bigquery_storage_v1.gapic import big_query_read_client # noqa from google.cloud.bigquery_storage_v1.proto import storage_pb2 from google.cloud.bigquery_storage_v1.proto import stream_pb2 diff --git a/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py b/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py index 82edfeb2..c1def1f9 100644 --- a/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py +++ b/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py @@ -19,7 +19,7 @@ import mock import pytest -from google.cloud.bigquery_storage_v1beta2.gapic import big_query_storage_client # noqa +from google.cloud.bigquery_storage_v1beta2.gapic import big_query_read_client # noqa from google.cloud.bigquery_storage_v1beta2.proto import storage_pb2 from google.cloud.bigquery_storage_v1beta2.proto import stream_pb2 From b625f60281ba7a3ca7a7477da27321a24633a1f0 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 19 Feb 2020 22:21:24 +0000 Subject: [PATCH 3/7] deal with stream pb2 in addition to storage pb2 --- google/cloud/bigquery_storage_v1/proto/stream_pb2.py | 10 +++++----- .../proto/table_reference_pb2.py | 4 ++-- .../cloud/bigquery_storage_v1beta2/proto/stream_pb2.py | 4 ++-- synth.metadata | 2 +- synth.py | 2 ++ 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/google/cloud/bigquery_storage_v1/proto/stream_pb2.py b/google/cloud/bigquery_storage_v1/proto/stream_pb2.py index 0d52a8de..333b7506 100644 --- a/google/cloud/bigquery_storage_v1/proto/stream_pb2.py +++ b/google/cloud/bigquery_storage_v1/proto/stream_pb2.py @@ -18,10 +18,10 @@ from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 from google.api import resource_pb2 as google_dot_api_dot_resource__pb2 -from google.cloud.bigquery.storage_v1.proto import ( +from google.cloud.bigquery_storage_v1.proto import ( arrow_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_arrow__pb2, ) -from google.cloud.bigquery.storage_v1.proto import ( +from google.cloud.bigquery_storage_v1.proto import ( avro_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1_dot_proto_dot_avro__pb2, ) from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 @@ -554,9 +554,9 @@ dict( DESCRIPTOR=_READSTREAM, __module__="google.cloud.bigquery.storage_v1.proto.stream_pb2", - __doc__="""Information about a single stream that gets data out of the storage - system. Most of the information about ``ReadStream`` instances is - aggregated, making ``ReadStream`` lightweight. + __doc__="""Information about a single stream that gets data out of + the storage system. Most of the information about ``ReadStream`` + instances is aggregated, making ``ReadStream`` lightweight. Attributes: diff --git a/google/cloud/bigquery_storage_v1beta1/proto/table_reference_pb2.py b/google/cloud/bigquery_storage_v1beta1/proto/table_reference_pb2.py index fd15501a..992067f0 100644 --- a/google/cloud/bigquery_storage_v1beta1/proto/table_reference_pb2.py +++ b/google/cloud/bigquery_storage_v1beta1/proto/table_reference_pb2.py @@ -162,8 +162,8 @@ dict( DESCRIPTOR=_TABLEREFERENCE, __module__="google.cloud.bigquery.storage_v1beta1.proto.table_reference_pb2", - __doc__="""Table reference that includes just the 3 strings needed to - identify a table. + __doc__="""Table reference that includes just the 3 strings needed to identify a + table. Attributes: diff --git a/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2.py b/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2.py index 1a597691..08e60bb3 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2.py +++ b/google/cloud/bigquery_storage_v1beta2/proto/stream_pb2.py @@ -18,10 +18,10 @@ from google.api import field_behavior_pb2 as google_dot_api_dot_field__behavior__pb2 from google.api import resource_pb2 as google_dot_api_dot_resource__pb2 -from google.cloud.bigquery.storage_v1beta2.proto import ( +from google.cloud.bigquery_storage_v1beta2.proto import ( arrow_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_arrow__pb2, ) -from google.cloud.bigquery.storage_v1beta2.proto import ( +from google.cloud.bigquery_storage_v1beta2.proto import ( avro_pb2 as google_dot_cloud_dot_bigquery_dot_storage__v1beta2_dot_proto_dot_avro__pb2, ) from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 diff --git a/synth.metadata b/synth.metadata index fedd0785..3d6c0fa2 100644 --- a/synth.metadata +++ b/synth.metadata @@ -1,5 +1,5 @@ { - "updateTime": "2020-02-19T20:23:03.706005Z", + "updateTime": "2020-02-19T22:20:30.216528Z", "sources": [ { "generator": { diff --git a/synth.py b/synth.py index f83db596..f2397f5c 100644 --- a/synth.py +++ b/synth.py @@ -78,6 +78,8 @@ [ f"google/cloud/bigquery_storage_{version}/proto/storage_pb2.py", f"google/cloud/bigquery_storage_{version}/proto/storage_pb2_grpc.py", + f"google/cloud/bigquery_storage_{version}/proto/stream_pb2.py", + f"google/cloud/bigquery_storage_{version}/proto/stream_pb2_grpc.py", ], f"from google.cloud.bigquery.storage_{version}.proto", f"from google.cloud.bigquery_storage_{version}.proto", From ef126d965dfa06c8a21455d97e09aac535cb2548 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 19 Feb 2020 22:51:19 +0000 Subject: [PATCH 4/7] another substition storage->read --- synth.metadata | 2 +- synth.py | 2 +- tests/unit/gapic/v1/test_big_query_read_client_v1.py | 12 ++++++------ .../v1beta2/test_big_query_read_client_v1beta2.py | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/synth.metadata b/synth.metadata index 3d6c0fa2..33b354ae 100644 --- a/synth.metadata +++ b/synth.metadata @@ -1,5 +1,5 @@ { - "updateTime": "2020-02-19T22:20:30.216528Z", + "updateTime": "2020-02-19T22:51:02.901471Z", "sources": [ { "generator": { diff --git a/synth.py b/synth.py index f2397f5c..336384b9 100644 --- a/synth.py +++ b/synth.py @@ -107,7 +107,7 @@ s.replace( [f"tests/unit/gapic/{version}/test_big_query_{clientinfo['type']}_client_{version}.py"], f"bigquery_storage_{version}.{clientinfo['name']}", - f"big_query_storage_client.{clientinfo['name']}", + f"big_query_{clientinfo['type']}_client.{clientinfo['name']}", ) # START: Ignore lint and coverage diff --git a/tests/unit/gapic/v1/test_big_query_read_client_v1.py b/tests/unit/gapic/v1/test_big_query_read_client_v1.py index 93d775ad..a82d5209 100644 --- a/tests/unit/gapic/v1/test_big_query_read_client_v1.py +++ b/tests/unit/gapic/v1/test_big_query_read_client_v1.py @@ -76,7 +76,7 @@ def test_create_read_session(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() response = client.create_read_session() assert expected_response == response @@ -92,7 +92,7 @@ def test_create_read_session_exception(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() with pytest.raises(CustomException): client.create_read_session() @@ -108,7 +108,7 @@ def test_read_rows(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() response = client.read_rows() resources = list(response) @@ -126,7 +126,7 @@ def test_read_rows_exception(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() with pytest.raises(CustomException): client.read_rows() @@ -141,7 +141,7 @@ def test_split_read_stream(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() response = client.split_read_stream() assert expected_response == response @@ -157,7 +157,7 @@ def test_split_read_stream_exception(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() with pytest.raises(CustomException): client.split_read_stream() diff --git a/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py b/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py index c1def1f9..ccac1ecd 100644 --- a/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py +++ b/tests/unit/gapic/v1beta2/test_big_query_read_client_v1beta2.py @@ -76,7 +76,7 @@ def test_create_read_session(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() response = client.create_read_session() assert expected_response == response @@ -92,7 +92,7 @@ def test_create_read_session_exception(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() with pytest.raises(CustomException): client.create_read_session() @@ -108,7 +108,7 @@ def test_read_rows(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() response = client.read_rows() resources = list(response) @@ -126,7 +126,7 @@ def test_read_rows_exception(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() with pytest.raises(CustomException): client.read_rows() @@ -141,7 +141,7 @@ def test_split_read_stream(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() response = client.split_read_stream() assert expected_response == response @@ -157,7 +157,7 @@ def test_split_read_stream_exception(self): patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel - client = big_query_storage_client.BigQueryReadClient() + client = big_query_read_client.BigQueryReadClient() with pytest.raises(CustomException): client.split_read_stream() From 97dd299569120bed7a12c105d81f886ccbd51c6f Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Wed, 19 Feb 2020 23:36:45 +0000 Subject: [PATCH 5/7] add gapic rst files to docs index toctree --- docs/index.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index f34b0cfb..8c0bb7eb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,6 +9,14 @@ API Reference gapic/v1beta1/reader gapic/v1beta1/types + gapic/v1beta2/api + gapic/v1beta2/reader + gapic/v1beta2/types + + gapic/v1/api + gapic/v1/reader + gapic/v1/types + Example Usage ------------- From 509c7c3de240258b2abd39b94bf57c8b0eef7508 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 20 Feb 2020 00:08:57 +0000 Subject: [PATCH 6/7] reader.rst only present in v1beta1 --- docs/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8c0bb7eb..07a7435c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,11 +10,9 @@ API Reference gapic/v1beta1/types gapic/v1beta2/api - gapic/v1beta2/reader gapic/v1beta2/types gapic/v1/api - gapic/v1/reader gapic/v1/types Example Usage From 1901e9c269a524a49354f19f10edae608078368d Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 20 Feb 2020 23:57:51 +0000 Subject: [PATCH 7/7] add empty __init__.py files to appease the 2.7 gods --- google/cloud/bigquery_storage_v1/__init__.py | 0 .../bigquery_storage_v1beta1/proto/table_reference_pb2.py | 4 ++-- google/cloud/bigquery_storage_v1beta2/__init__.py | 0 synth.metadata | 7 ++++--- 4 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 google/cloud/bigquery_storage_v1/__init__.py create mode 100644 google/cloud/bigquery_storage_v1beta2/__init__.py diff --git a/google/cloud/bigquery_storage_v1/__init__.py b/google/cloud/bigquery_storage_v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/google/cloud/bigquery_storage_v1beta1/proto/table_reference_pb2.py b/google/cloud/bigquery_storage_v1beta1/proto/table_reference_pb2.py index 992067f0..fd15501a 100644 --- a/google/cloud/bigquery_storage_v1beta1/proto/table_reference_pb2.py +++ b/google/cloud/bigquery_storage_v1beta1/proto/table_reference_pb2.py @@ -162,8 +162,8 @@ dict( DESCRIPTOR=_TABLEREFERENCE, __module__="google.cloud.bigquery.storage_v1beta1.proto.table_reference_pb2", - __doc__="""Table reference that includes just the 3 strings needed to identify a - table. + __doc__="""Table reference that includes just the 3 strings needed to + identify a table. Attributes: diff --git a/google/cloud/bigquery_storage_v1beta2/__init__.py b/google/cloud/bigquery_storage_v1beta2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/synth.metadata b/synth.metadata index 33b354ae..bef95ffe 100644 --- a/synth.metadata +++ b/synth.metadata @@ -1,5 +1,5 @@ { - "updateTime": "2020-02-19T22:51:02.901471Z", + "updateTime": "2020-02-20T23:57:11.494163Z", "sources": [ { "generator": { @@ -12,8 +12,9 @@ "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "a9639a0a9854fd6e1be08bba1ac3897f4f16cb2f", - "internalRef": "295983266" + "sha": "3eaaaf8626ce5b0c0bc7eee05e143beffa373b01", + "internalRef": "296274723", + "log": "3eaaaf8626ce5b0c0bc7eee05e143beffa373b01\nAdd BUILD.bazel for v1 secretmanager.googleapis.com\n\nPiperOrigin-RevId: 296274723\n\ne76149c3d992337f85eeb45643106aacae7ede82\nMove securitycenter v1 to use generate from annotations.\n\nPiperOrigin-RevId: 296266862\n\n203740c78ac69ee07c3bf6be7408048751f618f8\nAdd StackdriverLoggingConfig field to Cloud Tasks v2 API.\n\nPiperOrigin-RevId: 296256388\n\n" } }, {