Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit d910a89

Browse files
yoshi-automationyoshi-automation
authored
feat: Add ZSTD compression as an option for Arrow proto changes
* feat: Add ZSTD compression as an option for Arrow. Committer: @emkornfield PiperOrigin-RevId: 374220891 Source-Author: Google APIs <noreply@google.com> Source-Date: Mon May 17 10:03:14 2021 -0700 Source-Repo: googleapis/googleapis Source-Sha: 23efea9fc7bedfe53b24295ed84b5f873606edcb Source-Link: googleapis/googleapis@23efea9 * chore: release gapic-generator-java v1.0.5 Committer: @miraleung PiperOrigin-RevId: 374252908 Source-Author: Google APIs <noreply@google.com> Source-Date: Mon May 17 12:23:32 2021 -0700 Source-Repo: googleapis/googleapis Source-Sha: 131ae3e375c05856d7d77cd146dc2af92650eb38 Source-Link: googleapis/googleapis@131ae3e
1 parent 1fcc01d commit d910a89

File tree

10 files changed

+275
-132
lines changed

10 files changed

+275
-132
lines changed

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta1/BaseBigQueryStorageClient.java‎

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -263,8 +263,13 @@ public final Storage.ReadSession createReadSession(
263263
* try (BaseBigQueryStorageClient baseBigQueryStorageClient = BaseBigQueryStorageClient.create()) {
264264
* Storage.CreateReadSessionRequest request =
265265
* Storage.CreateReadSessionRequest.newBuilder()
266+
* .setTableReference(TableReferenceProto.TableReference.newBuilder().build())
266267
* .setParent(ProjectName.of("[PROJECT]").toString())
268+
* .setTableModifiers(TableReferenceProto.TableModifiers.newBuilder().build())
267269
* .setRequestedStreams(1017221410)
270+
* .setReadOptions(ReadOptions.TableReadOptions.newBuilder().build())
271+
* .setFormat(Storage.DataFormat.forNumber(0))
272+
* .setShardingStrategy(Storage.ShardingStrategy.forNumber(0))
268273
* .build();
269274
* Storage.ReadSession response = baseBigQueryStorageClient.createReadSession(request);
270275
* }
@@ -296,8 +301,13 @@ public final Storage.ReadSession createReadSession(Storage.CreateReadSessionRequ
296301
* try (BaseBigQueryStorageClient baseBigQueryStorageClient = BaseBigQueryStorageClient.create()) {
297302
* Storage.CreateReadSessionRequest request =
298303
* Storage.CreateReadSessionRequest.newBuilder()
304+
* .setTableReference(TableReferenceProto.TableReference.newBuilder().build())
299305
* .setParent(ProjectName.of("[PROJECT]").toString())
306+
* .setTableModifiers(TableReferenceProto.TableModifiers.newBuilder().build())
300307
* .setRequestedStreams(1017221410)
308+
* .setReadOptions(ReadOptions.TableReadOptions.newBuilder().build())
309+
* .setFormat(Storage.DataFormat.forNumber(0))
310+
* .setShardingStrategy(Storage.ShardingStrategy.forNumber(0))
301311
* .build();
302312
* ApiFuture<Storage.ReadSession> future =
303313
* baseBigQueryStorageClient.createReadSessionCallable().futureCall(request);
@@ -326,7 +336,10 @@ public final Storage.ReadSession createReadSession(Storage.CreateReadSessionRequ
326336
*
327337
* <pre>{@code
328338
* try (BaseBigQueryStorageClient baseBigQueryStorageClient = BaseBigQueryStorageClient.create()) {
329-
* Storage.ReadRowsRequest request = Storage.ReadRowsRequest.newBuilder().build();
339+
* Storage.ReadRowsRequest request =
340+
* Storage.ReadRowsRequest.newBuilder()
341+
* .setReadPosition(Storage.StreamPosition.newBuilder().build())
342+
* .build();
330343
* ServerStream<Storage.ReadRowsResponse> stream =
331344
* baseBigQueryStorageClient.readRowsCallable().call(request);
332345
* for (Storage.ReadRowsResponse response : stream) {
@@ -383,6 +396,7 @@ public final Storage.BatchCreateReadSessionStreamsResponse batchCreateReadSessio
383396
* try (BaseBigQueryStorageClient baseBigQueryStorageClient = BaseBigQueryStorageClient.create()) {
384397
* Storage.BatchCreateReadSessionStreamsRequest request =
385398
* Storage.BatchCreateReadSessionStreamsRequest.newBuilder()
399+
* .setSession(Storage.ReadSession.newBuilder().build())
386400
* .setRequestedStreams(1017221410)
387401
* .build();
388402
* Storage.BatchCreateReadSessionStreamsResponse response =
@@ -409,6 +423,7 @@ public final Storage.BatchCreateReadSessionStreamsResponse batchCreateReadSessio
409423
* try (BaseBigQueryStorageClient baseBigQueryStorageClient = BaseBigQueryStorageClient.create()) {
410424
* Storage.BatchCreateReadSessionStreamsRequest request =
411425
* Storage.BatchCreateReadSessionStreamsRequest.newBuilder()
426+
* .setSession(Storage.ReadSession.newBuilder().build())
412427
* .setRequestedStreams(1017221410)
413428
* .build();
414429
* ApiFuture<Storage.BatchCreateReadSessionStreamsResponse> future =
@@ -475,7 +490,10 @@ public final void finalizeStream(Storage.Stream stream) {
475490
*
476491
* <pre>{@code
477492
* try (BaseBigQueryStorageClient baseBigQueryStorageClient = BaseBigQueryStorageClient.create()) {
478-
* Storage.FinalizeStreamRequest request = Storage.FinalizeStreamRequest.newBuilder().build();
493+
* Storage.FinalizeStreamRequest request =
494+
* Storage.FinalizeStreamRequest.newBuilder()
495+
* .setStream(Storage.Stream.newBuilder().build())
496+
* .build();
479497
* baseBigQueryStorageClient.finalizeStream(request);
480498
* }
481499
* }</pre>
@@ -505,7 +523,10 @@ public final void finalizeStream(Storage.FinalizeStreamRequest request) {
505523
*
506524
* <pre>{@code
507525
* try (BaseBigQueryStorageClient baseBigQueryStorageClient = BaseBigQueryStorageClient.create()) {
508-
* Storage.FinalizeStreamRequest request = Storage.FinalizeStreamRequest.newBuilder().build();
526+
* Storage.FinalizeStreamRequest request =
527+
* Storage.FinalizeStreamRequest.newBuilder()
528+
* .setStream(Storage.Stream.newBuilder().build())
529+
* .build();
509530
* ApiFuture<Empty> future =
510531
* baseBigQueryStorageClient.finalizeStreamCallable().futureCall(request);
511532
* // Do something.
@@ -569,7 +590,10 @@ public final Storage.SplitReadStreamResponse splitReadStream(Storage.Stream orig
569590
* <pre>{@code
570591
* try (BaseBigQueryStorageClient baseBigQueryStorageClient = BaseBigQueryStorageClient.create()) {
571592
* Storage.SplitReadStreamRequest request =
572-
* Storage.SplitReadStreamRequest.newBuilder().setFraction(-1653751294).build();
593+
* Storage.SplitReadStreamRequest.newBuilder()
594+
* .setOriginalStream(Storage.Stream.newBuilder().build())
595+
* .setFraction(-1653751294)
596+
* .build();
573597
* Storage.SplitReadStreamResponse response = baseBigQueryStorageClient.splitReadStream(request);
574598
* }
575599
* }</pre>
@@ -601,7 +625,10 @@ public final Storage.SplitReadStreamResponse splitReadStream(
601625
* <pre>{@code
602626
* try (BaseBigQueryStorageClient baseBigQueryStorageClient = BaseBigQueryStorageClient.create()) {
603627
* Storage.SplitReadStreamRequest request =
604-
* Storage.SplitReadStreamRequest.newBuilder().setFraction(-1653751294).build();
628+
* Storage.SplitReadStreamRequest.newBuilder()
629+
* .setOriginalStream(Storage.Stream.newBuilder().build())
630+
* .setFraction(-1653751294)
631+
* .build();
605632
* ApiFuture<Storage.SplitReadStreamResponse> future =
606633
* baseBigQueryStorageClient.splitReadStreamCallable().futureCall(request);
607634
* // Do something.

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta1/BaseBigQueryStorageClientTest.java‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@
2828
import com.google.api.gax.rpc.StatusCode;
2929
import com.google.protobuf.AbstractMessage;
3030
import com.google.protobuf.Empty;
31+
import com.google.protobuf.Timestamp;
3132
import io.grpc.StatusRuntimeException;
3233
import java.io.IOException;
34+
import java.util.ArrayList;
3335
import java.util.Arrays;
3436
import java.util.List;
3537
import java.util.UUID;
@@ -82,7 +84,15 @@ public void tearDown() throws Exception {
8284

8385
@Test
8486
public void createReadSessionTest() throws Exception {
85-
Storage.ReadSession expectedResponse = Storage.ReadSession.newBuilder().build();
87+
Storage.ReadSession expectedResponse =
88+
Storage.ReadSession.newBuilder()
89+
.setName(ReadSessionName.of("[PROJECT]", "[LOCATION]", "[SESSION]").toString())
90+
.setExpireTime(Timestamp.newBuilder().build())
91+
.addAllStreams(new ArrayList<Storage.Stream>())
92+
.setTableReference(TableReferenceProto.TableReference.newBuilder().build())
93+
.setTableModifiers(TableReferenceProto.TableModifiers.newBuilder().build())
94+
.setShardingStrategy(Storage.ShardingStrategy.forNumber(0))
95+
.build();
8696
mockBigQueryStorage.addResponse(expectedResponse);
8797

8898
TableReferenceProto.TableReference tableReference =
@@ -127,7 +137,15 @@ public void createReadSessionExceptionTest() throws Exception {
127137

128138
@Test
129139
public void createReadSessionTest2() throws Exception {
130-
Storage.ReadSession expectedResponse = Storage.ReadSession.newBuilder().build();
140+
Storage.ReadSession expectedResponse =
141+
Storage.ReadSession.newBuilder()
142+
.setName(ReadSessionName.of("[PROJECT]", "[LOCATION]", "[SESSION]").toString())
143+
.setExpireTime(Timestamp.newBuilder().build())
144+
.addAllStreams(new ArrayList<Storage.Stream>())
145+
.setTableReference(TableReferenceProto.TableReference.newBuilder().build())
146+
.setTableModifiers(TableReferenceProto.TableModifiers.newBuilder().build())
147+
.setShardingStrategy(Storage.ShardingStrategy.forNumber(0))
148+
.build();
131149
mockBigQueryStorage.addResponse(expectedResponse);
132150

133151
TableReferenceProto.TableReference tableReference =
@@ -172,9 +190,17 @@ public void createReadSessionExceptionTest2() throws Exception {
172190

173191
@Test
174192
public void readRowsTest() throws Exception {
175-
Storage.ReadRowsResponse expectedResponse = Storage.ReadRowsResponse.newBuilder().build();
193+
Storage.ReadRowsResponse expectedResponse =
194+
Storage.ReadRowsResponse.newBuilder()
195+
.setRowCount(1340416618)
196+
.setStatus(Storage.StreamStatus.newBuilder().build())
197+
.setThrottleStatus(Storage.ThrottleStatus.newBuilder().build())
198+
.build();
176199
mockBigQueryStorage.addResponse(expectedResponse);
177-
Storage.ReadRowsRequest request = Storage.ReadRowsRequest.newBuilder().build();
200+
Storage.ReadRowsRequest request =
201+
Storage.ReadRowsRequest.newBuilder()
202+
.setReadPosition(Storage.StreamPosition.newBuilder().build())
203+
.build();
178204

179205
MockStreamObserver<Storage.ReadRowsResponse> responseObserver = new MockStreamObserver<>();
180206

@@ -191,7 +217,10 @@ public void readRowsTest() throws Exception {
191217
public void readRowsExceptionTest() throws Exception {
192218
StatusRuntimeException exception = new StatusRuntimeException(io.grpc.Status.INVALID_ARGUMENT);
193219
mockBigQueryStorage.addException(exception);
194-
Storage.ReadRowsRequest request = Storage.ReadRowsRequest.newBuilder().build();
220+
Storage.ReadRowsRequest request =
221+
Storage.ReadRowsRequest.newBuilder()
222+
.setReadPosition(Storage.StreamPosition.newBuilder().build())
223+
.build();
195224

196225
MockStreamObserver<Storage.ReadRowsResponse> responseObserver = new MockStreamObserver<>();
197226

@@ -212,7 +241,9 @@ public void readRowsExceptionTest() throws Exception {
212241
@Test
213242
public void batchCreateReadSessionStreamsTest() throws Exception {
214243
Storage.BatchCreateReadSessionStreamsResponse expectedResponse =
215-
Storage.BatchCreateReadSessionStreamsResponse.newBuilder().build();
244+
Storage.BatchCreateReadSessionStreamsResponse.newBuilder()
245+
.addAllStreams(new ArrayList<Storage.Stream>())
246+
.build();
216247
mockBigQueryStorage.addResponse(expectedResponse);
217248

218249
Storage.ReadSession session = Storage.ReadSession.newBuilder().build();
@@ -288,7 +319,10 @@ public void finalizeStreamExceptionTest() throws Exception {
288319
@Test
289320
public void splitReadStreamTest() throws Exception {
290321
Storage.SplitReadStreamResponse expectedResponse =
291-
Storage.SplitReadStreamResponse.newBuilder().build();
322+
Storage.SplitReadStreamResponse.newBuilder()
323+
.setPrimaryStream(Storage.Stream.newBuilder().build())
324+
.setRemainderStream(Storage.Stream.newBuilder().build())
325+
.build();
292326
mockBigQueryStorage.addResponse(expectedResponse);
293327

294328
Storage.Stream originalStream = Storage.Stream.newBuilder().build();

‎proto-google-cloud-bigquerystorage-v1/src/main/java/com/google/cloud/bigquery/storage/v1/ArrowProto.java‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,17 +52,18 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
5252
+ ".proto\022 google.cloud.bigquery.storage.v1"
5353
+ "\"(\n\013ArrowSchema\022\031\n\021serialized_schema\030\001 \001"
5454
+ "(\014\"F\n\020ArrowRecordBatch\022\037\n\027serialized_rec"
55-
+ "ord_batch\030\001 \001(\014\022\021\n\trow_count\030\002 \001(\003\"\305\001\n\031A"
55+
+ "ord_batch\030\001 \001(\014\022\021\n\trow_count\030\002 \001(\003\"\317\001\n\031A"
5656
+ "rrowSerializationOptions\022h\n\022buffer_compr"
5757
+ "ession\030\002 \001(\0162L.google.cloud.bigquery.sto"
5858
+ "rage.v1.ArrowSerializationOptions.Compre"
59-
+ "ssionCodec\">\n\020CompressionCodec\022\033\n\027COMPRE"
60-
+ "SSION_UNSPECIFIED\020\000\022\r\n\tLZ4_FRAME\020\001B\303\001\n$c"
61-
+ "om.google.cloud.bigquery.storage.v1B\nArr"
62-
+ "owProtoP\001ZGgoogle.golang.org/genproto/go"
63-
+ "ogleapis/cloud/bigquery/storage/v1;stora"
64-
+ "ge\252\002 Google.Cloud.BigQuery.Storage.V1\312\002 "
65-
+ "Google\\Cloud\\BigQuery\\Storage\\V1b\006proto3"
59+
+ "ssionCodec\"H\n\020CompressionCodec\022\033\n\027COMPRE"
60+
+ "SSION_UNSPECIFIED\020\000\022\r\n\tLZ4_FRAME\020\001\022\010\n\004ZS"
61+
+ "TD\020\002B\303\001\n$com.google.cloud.bigquery.stora"
62+
+ "ge.v1B\nArrowProtoP\001ZGgoogle.golang.org/g"
63+
+ "enproto/googleapis/cloud/bigquery/storag"
64+
+ "e/v1;storage\252\002 Google.Cloud.BigQuery.Sto"
65+
+ "rage.V1\312\002 Google\\Cloud\\BigQuery\\Storage\\"
66+
+ "V1b\006proto3"
6667
};
6768
descriptor =
6869
com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(

‎proto-google-cloud-bigquerystorage-v1/src/main/java/com/google/cloud/bigquery/storage/v1/ArrowSerializationOptions.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,16 @@ public enum CompressionCodec implements com.google.protobuf.ProtocolMessageEnum
142142
* <code>LZ4_FRAME = 1;</code>
143143
*/
144144
LZ4_FRAME(1),
145+
/**
146+
*
147+
*
148+
* <pre>
149+
* Zstandard compression.
150+
* </pre>
151+
*
152+
* <code>ZSTD = 2;</code>
153+
*/
154+
ZSTD(2),
145155
UNRECOGNIZED(-1),
146156
;
147157

@@ -165,6 +175,16 @@ public enum CompressionCodec implements com.google.protobuf.ProtocolMessageEnum
165175
* <code>LZ4_FRAME = 1;</code>
166176
*/
167177
public static final int LZ4_FRAME_VALUE = 1;
178+
/**
179+
*
180+
*
181+
* <pre>
182+
* Zstandard compression.
183+
* </pre>
184+
*
185+
* <code>ZSTD = 2;</code>
186+
*/
187+
public static final int ZSTD_VALUE = 2;
168188

169189
public final int getNumber() {
170190
if (this == UNRECOGNIZED) {
@@ -194,6 +214,8 @@ public static CompressionCodec forNumber(int value) {
194214
return COMPRESSION_UNSPECIFIED;
195215
case 1:
196216
return LZ4_FRAME;
217+
case 2:
218+
return ZSTD;
197219
default:
198220
return null;
199221
}

0 commit comments

Comments
 (0)