Skip to content
Permalink
Browse files
feat: add BigQuery Storage Write API v1 (#301)
* feat: add BigQuery Storage Write API v1

Committer: @yirutang
PiperOrigin-RevId: 397350004

Source-Link: googleapis/googleapis@b4da4fd

Source-Link: https://github.com/googleapis/googleapis-gen/commit/67bcfcfa00a4113e968bac1a10d0ad0c1b7dc45b
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNjdiY2ZjZmEwMGE0MTEzZTk2OGJhYzFhMTBkMGFkMGMxYjdkYzQ1YiJ9

* 🦉 Updates from OwlBot

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
  • Loading branch information
gcf-owl-bot[bot] and gcf-owl-bot[bot] committed Sep 17, 2021
1 parent eb74af9 commit 69e3fb8ec2ecac0417b6a4bc954004a064ae04b7
Showing with 5,842 additions and 30 deletions.
  1. +6 −0 docs/bigquery_storage_v1/big_query_write.rst
  2. +1 −0 docs/bigquery_storage_v1/services.rst
  3. +44 −0 google/cloud/bigquery_storage/__init__.py
  4. +74 −0 google/cloud/bigquery_storage_v1/gapic_metadata.json
  5. +2 −12 google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py
  6. +1 −1 google/cloud/bigquery_storage_v1/services/big_query_read/client.py
  7. +1 −13 google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py
  8. +1 −1 google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py
  9. +1 −1 google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py
  10. +22 −0 google/cloud/bigquery_storage_v1/services/big_query_write/__init__.py
  11. +714 −0 google/cloud/bigquery_storage_v1/services/big_query_write/async_client.py
  12. +855 −0 google/cloud/bigquery_storage_v1/services/big_query_write/client.py
  13. +33 −0 google/cloud/bigquery_storage_v1/services/big_query_write/transports/__init__.py
  14. +314 −0 google/cloud/bigquery_storage_v1/services/big_query_write/transports/base.py
  15. +448 −0 google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc.py
  16. +452 −0 google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc_asyncio.py
  17. +36 −0 google/cloud/bigquery_storage_v1/types/__init__.py
  18. +64 −0 google/cloud/bigquery_storage_v1/types/protobuf.py
  19. +282 −1 google/cloud/bigquery_storage_v1/types/storage.py
  20. +49 −1 google/cloud/bigquery_storage_v1/types/stream.py
  21. +149 −0 google/cloud/bigquery_storage_v1/types/table.py
  22. +6 −0 scripts/fixup_bigquery_storage_v1_keywords.py
  23. +4 −0 tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py
  24. +2,283 −0 tests/unit/gapic/bigquery_storage_v1/test_big_query_write.py
@@ -0,0 +1,6 @@
BigQueryWrite
-------------------------------

.. automodule:: google.cloud.bigquery_storage_v1.services.big_query_write
:members:
:inherited-members:
@@ -4,3 +4,4 @@ Services for Google Cloud Bigquery Storage v1 API
:maxdepth: 2

big_query_read
big_query_write
@@ -15,6 +15,12 @@
#

from google.cloud.bigquery_storage_v1 import BigQueryReadClient
from google.cloud.bigquery_storage_v1.services.big_query_write.client import (
BigQueryWriteClient,
)
from google.cloud.bigquery_storage_v1.services.big_query_write.async_client import (
BigQueryWriteAsyncClient,
)

from google.cloud.bigquery_storage_v1 import gapic_types as types
from google.cloud.bigquery_storage_v1 import __version__
@@ -23,34 +29,72 @@
from google.cloud.bigquery_storage_v1.types.arrow import ArrowSerializationOptions
from google.cloud.bigquery_storage_v1.types.avro import AvroRows
from google.cloud.bigquery_storage_v1.types.avro import AvroSchema
from google.cloud.bigquery_storage_v1.types.protobuf import ProtoRows
from google.cloud.bigquery_storage_v1.types.protobuf import ProtoSchema
from google.cloud.bigquery_storage_v1.types.storage import AppendRowsRequest
from google.cloud.bigquery_storage_v1.types.storage import AppendRowsResponse
from google.cloud.bigquery_storage_v1.types.storage import (
BatchCommitWriteStreamsRequest,
)
from google.cloud.bigquery_storage_v1.types.storage import (
BatchCommitWriteStreamsResponse,
)
from google.cloud.bigquery_storage_v1.types.storage import CreateReadSessionRequest
from google.cloud.bigquery_storage_v1.types.storage import CreateWriteStreamRequest
from google.cloud.bigquery_storage_v1.types.storage import FinalizeWriteStreamRequest
from google.cloud.bigquery_storage_v1.types.storage import FinalizeWriteStreamResponse
from google.cloud.bigquery_storage_v1.types.storage import FlushRowsRequest
from google.cloud.bigquery_storage_v1.types.storage import FlushRowsResponse
from google.cloud.bigquery_storage_v1.types.storage import GetWriteStreamRequest
from google.cloud.bigquery_storage_v1.types.storage import ReadRowsRequest
from google.cloud.bigquery_storage_v1.types.storage import ReadRowsResponse
from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamRequest
from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamResponse
from google.cloud.bigquery_storage_v1.types.storage import StorageError
from google.cloud.bigquery_storage_v1.types.storage import StreamStats
from google.cloud.bigquery_storage_v1.types.storage import ThrottleState
from google.cloud.bigquery_storage_v1.types.stream import ReadSession
from google.cloud.bigquery_storage_v1.types.stream import ReadStream
from google.cloud.bigquery_storage_v1.types.stream import WriteStream
from google.cloud.bigquery_storage_v1.types.stream import DataFormat
from google.cloud.bigquery_storage_v1.types.table import TableFieldSchema
from google.cloud.bigquery_storage_v1.types.table import TableSchema

__all__ = (
"BigQueryReadClient",
"BigQueryWriteClient",
"BigQueryWriteAsyncClient",
"__version__",
"types",
"ArrowRecordBatch",
"ArrowSchema",
"ArrowSerializationOptions",
"AvroRows",
"AvroSchema",
"ProtoRows",
"ProtoSchema",
"AppendRowsRequest",
"AppendRowsResponse",
"BatchCommitWriteStreamsRequest",
"BatchCommitWriteStreamsResponse",
"CreateReadSessionRequest",
"CreateWriteStreamRequest",
"FinalizeWriteStreamRequest",
"FinalizeWriteStreamResponse",
"FlushRowsRequest",
"FlushRowsResponse",
"GetWriteStreamRequest",
"ReadRowsRequest",
"ReadRowsResponse",
"SplitReadStreamRequest",
"SplitReadStreamResponse",
"StorageError",
"StreamStats",
"ThrottleState",
"ReadSession",
"ReadStream",
"WriteStream",
"DataFormat",
"TableFieldSchema",
"TableSchema",
)
@@ -48,6 +48,80 @@
}
}
}
},
"BigQueryWrite": {
"clients": {
"grpc": {
"libraryClient": "BigQueryWriteClient",
"rpcs": {
"AppendRows": {
"methods": [
"append_rows"
]
},
"BatchCommitWriteStreams": {
"methods": [
"batch_commit_write_streams"
]
},
"CreateWriteStream": {
"methods": [
"create_write_stream"
]
},
"FinalizeWriteStream": {
"methods": [
"finalize_write_stream"
]
},
"FlushRows": {
"methods": [
"flush_rows"
]
},
"GetWriteStream": {
"methods": [
"get_write_stream"
]
}
}
},
"grpc-async": {
"libraryClient": "BigQueryWriteAsyncClient",
"rpcs": {
"AppendRows": {
"methods": [
"append_rows"
]
},
"BatchCommitWriteStreams": {
"methods": [
"batch_commit_write_streams"
]
},
"CreateWriteStream": {
"methods": [
"create_write_stream"
]
},
"FinalizeWriteStream": {
"methods": [
"finalize_write_stream"
]
},
"FlushRows": {
"methods": [
"flush_rows"
]
},
"GetWriteStream": {
"methods": [
"get_write_stream"
]
}
}
}
}
}
}
}
@@ -197,7 +197,7 @@ async def create_read_session(
number of pre-filtered rows, so some filters can lead to
lopsided assignments.
Read sessions automatically expire 24 hours after they
Read sessions automatically expire 6 hours after they
are created and do not require manual clean-up by the
caller.
@@ -441,17 +441,7 @@ async def split_read_stream(
# and friendly error handling.
rpc = gapic_v1.method_async.wrap_method(
self._client._transport.split_read_stream,
default_retry=retries.Retry(
initial=0.1,
maximum=60.0,
multiplier=1.3,
predicate=retries.if_exception_type(
core_exceptions.DeadlineExceeded,
core_exceptions.ServiceUnavailable,
),
deadline=600.0,
),
default_timeout=600.0,
default_timeout=None,
client_info=DEFAULT_CLIENT_INFO,
)

@@ -418,7 +418,7 @@ def create_read_session(
number of pre-filtered rows, so some filters can lead to
lopsided assignments.
Read sessions automatically expire 24 hours after they
Read sessions automatically expire 6 hours after they
are created and do not require manual clean-up by the
caller.
@@ -189,19 +189,7 @@ def _prep_wrapped_messages(self, client_info):
client_info=client_info,
),
self.split_read_stream: gapic_v1.method.wrap_method(
self.split_read_stream,
default_retry=retries.Retry(
initial=0.1,
maximum=60.0,
multiplier=1.3,
predicate=retries.if_exception_type(
core_exceptions.DeadlineExceeded,
core_exceptions.ServiceUnavailable,
),
deadline=600.0,
),
default_timeout=600.0,
client_info=client_info,
self.split_read_stream, default_timeout=None, client_info=client_info,
),
}

@@ -253,7 +253,7 @@ def create_read_session(
number of pre-filtered rows, so some filters can lead to
lopsided assignments.
Read sessions automatically expire 24 hours after they
Read sessions automatically expire 6 hours after they
are created and do not require manual clean-up by the
caller.
@@ -256,7 +256,7 @@ def create_read_session(
number of pre-filtered rows, so some filters can lead to
lopsided assignments.
Read sessions automatically expire 24 hours after they
Read sessions automatically expire 6 hours after they
are created and do not require manual clean-up by the
caller.
@@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from .client import BigQueryWriteClient
from .async_client import BigQueryWriteAsyncClient

__all__ = (
"BigQueryWriteClient",
"BigQueryWriteAsyncClient",
)

0 comments on commit 69e3fb8

Please sign in to comment.