Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit 0988105

Browse files
stephaniewang526yirutang
stephaniewang526
and
yirutang
authored
feat: remove IgnoreUnknownFields support on JsonStreamWriter (#757)
* feat! Remove IgnoreUnknownFields support for JsonStreamWriter * . * . * . * . * . * . * . * . * . * add clirr-ignored-differences.xml for breaking changes Co-authored-by: yirutang <yiru@google.com>
1 parent f3c897f commit 0988105

File tree

8 files changed

+136
-187
lines changed

8 files changed

+136
-187
lines changed

‎google-cloud-bigquerystorage/clirr-ignored-differences.xml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!-- see http://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
3+
<differences>
4+
<!-- TODO(stephwang): To be removed after the release -->
5+
<difference>
6+
<className>com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter</className>
7+
<differenceType>7005</differenceType>
8+
<method>com.google.api.core.ApiFuture append(org.json.JSONArray, boolean)</method>
9+
<from>com.google.api.core.ApiFuture append(org.json.JSONArray, boolean)</from>
10+
<to>com.google.api.core.ApiFuture append(org.json.JSONArray, long)</to>
11+
</difference>
12+
<difference>
13+
<className>com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter</className>
14+
<differenceType>7004</differenceType>
15+
<method>com.google.api.core.ApiFuture append(org.json.JSONArray, long, boolean)</method>
16+
<from>com.google.api.core.ApiFuture append(org.json.JSONArray, long, boolean)</from>
17+
<to>com.google.api.core.ApiFuture append(org.json.JSONArray)</to>
18+
</difference>
19+
<difference>
20+
<className>com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage</className>
21+
<differenceType>7004</differenceType>
22+
<method>com.google.protobuf.DynamicMessage convertJsonToProtoMessage(com.google.protobuf.Descriptors$Descriptor, org.json.JSONObject, boolean)</method>
23+
<from>com.google.protobuf.DynamicMessage convertJsonToProtoMessage(com.google.protobuf.Descriptors$Descriptor, org.json.JSONObject, boolean)</from>
24+
<to>com.google.protobuf.DynamicMessage convertJsonToProtoMessage(com.google.protobuf.Descriptors$Descriptor, org.json.JSONObject)</to>
25+
</difference>
26+
</differences>

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.java‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -94,12 +94,11 @@ private JsonStreamWriter(Builder builder)
9494
* schema update, the OnSchemaUpdateRunnable will be used to determine what actions to perform.
9595
*
9696
* @param jsonArr The JSON array that contains JSONObjects to be written
97-
* @param allowUnknownFields if true, json data can have fields unknown to the BigQuery table.
9897
* @return ApiFuture<AppendRowsResponse> returns an AppendRowsResponse message wrapped in an
9998
* ApiFuture
10099
*/
101-
public ApiFuture<AppendRowsResponse> append(JSONArray jsonArr, boolean allowUnknownFields) {
102-
return append(jsonArr, -1, allowUnknownFields);
100+
public ApiFuture<AppendRowsResponse> append(JSONArray jsonArr) {
101+
return append(jsonArr, -1);
103102
}
104103

105104
/**
@@ -109,20 +108,17 @@ public ApiFuture<AppendRowsResponse> append(JSONArray jsonArr, boolean allowUnkn
109108
*
110109
* @param jsonArr The JSON array that contains JSONObjects to be written
111110
* @param offset Offset for deduplication
112-
* @param allowUnknownFields if true, json data can have fields unknown to the BigQuery table.
113111
* @return ApiFuture<AppendRowsResponse> returns an AppendRowsResponse message wrapped in an
114112
* ApiFuture
115113
*/
116-
public ApiFuture<AppendRowsResponse> append(
117-
JSONArray jsonArr, long offset, boolean allowUnknownFields) {
114+
public ApiFuture<AppendRowsResponse> append(JSONArray jsonArr, long offset) {
118115
ProtoRows.Builder rowsBuilder = ProtoRows.newBuilder();
119116
// Any error in convertJsonToProtoMessage will throw an
120117
// IllegalArgumentException/IllegalStateException/NullPointerException and will halt processing
121118
// of JSON data.
122119
for (int i = 0; i < jsonArr.length(); i++) {
123120
JSONObject json = jsonArr.getJSONObject(i);
124-
Message protoMessage =
125-
JsonToProtoMessage.convertJsonToProtoMessage(this.descriptor, json, allowUnknownFields);
121+
Message protoMessage = JsonToProtoMessage.convertJsonToProtoMessage(this.descriptor, json);
126122
rowsBuilder.addSerializedRows(protoMessage.toByteString());
127123
}
128124
AppendRowsRequest.ProtoData.Builder data = AppendRowsRequest.ProtoData.newBuilder();

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage.java‎

Lines changed: 11 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -47,18 +47,15 @@ public class JsonToProtoMessage {
4747
*
4848
* @param protoSchema
4949
* @param json
50-
* @param allowUnknownFields Ignores unknown JSON fields.
5150
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
5251
*/
53-
public static DynamicMessage convertJsonToProtoMessage(
54-
Descriptor protoSchema, JSONObject json, boolean allowUnknownFields)
52+
public static DynamicMessage convertJsonToProtoMessage(Descriptor protoSchema, JSONObject json)
5553
throws IllegalArgumentException {
5654
Preconditions.checkNotNull(json, "JSONObject is null.");
5755
Preconditions.checkNotNull(protoSchema, "Protobuf descriptor is null.");
5856
Preconditions.checkState(json.length() != 0, "JSONObject is empty.");
5957

60-
return convertJsonToProtoMessageImpl(
61-
protoSchema, json, "root", /*topLevel=*/ true, allowUnknownFields);
58+
return convertJsonToProtoMessageImpl(protoSchema, json, "root", /*topLevel=*/ true);
6259
}
6360

6461
/**
@@ -67,24 +64,18 @@ public static DynamicMessage convertJsonToProtoMessage(
6764
* @param protoSchema
6865
* @param json
6966
* @param jsonScope Debugging purposes
70-
* @param allowUnknownFields Ignores unknown JSON fields.
7167
* @param topLevel checks if root level has any matching fields.
7268
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
7369
*/
7470
private static DynamicMessage convertJsonToProtoMessageImpl(
75-
Descriptor protoSchema,
76-
JSONObject json,
77-
String jsonScope,
78-
boolean topLevel,
79-
boolean allowUnknownFields)
71+
Descriptor protoSchema, JSONObject json, String jsonScope, boolean topLevel)
8072
throws IllegalArgumentException {
8173

8274
DynamicMessage.Builder protoMsg = DynamicMessage.newBuilder(protoSchema);
8375
String[] jsonNames = JSONObject.getNames(json);
8476
if (jsonNames == null) {
8577
return protoMsg.build();
8678
}
87-
int matchedFields = 0;
8879
for (int i = 0; i < jsonNames.length; i++) {
8980
String jsonName = jsonNames[i];
9081
// We want lowercase here to support case-insensitive data writes.
@@ -93,27 +84,16 @@ private static DynamicMessage convertJsonToProtoMessageImpl(
9384
String currentScope = jsonScope + "." + jsonName;
9485
FieldDescriptor field = protoSchema.findFieldByName(jsonLowercaseName);
9586
if (field == null) {
96-
if (!allowUnknownFields) {
97-
throw new IllegalArgumentException(
98-
String.format(
99-
"JSONObject has fields unknown to BigQuery: %s. Set allowUnknownFields to True to allow unknown fields.",
100-
currentScope));
101-
} else {
102-
continue;
103-
}
87+
throw new IllegalArgumentException(
88+
String.format("JSONObject has fields unknown to BigQuery: %s.", currentScope));
10489
}
105-
matchedFields++;
10690
if (!field.isRepeated()) {
107-
fillField(protoMsg, field, json, jsonName, currentScope, allowUnknownFields);
91+
fillField(protoMsg, field, json, jsonName, currentScope);
10892
} else {
109-
fillRepeatedField(protoMsg, field, json, jsonName, currentScope, allowUnknownFields);
93+
fillRepeatedField(protoMsg, field, json, jsonName, currentScope);
11094
}
11195
}
11296

113-
if (matchedFields == 0 && topLevel) {
114-
throw new IllegalArgumentException(
115-
"There are no matching fields found for the JSONObject and the protocol buffer descriptor.");
116-
}
11797
DynamicMessage msg;
11898
try {
11999
msg = protoMsg.build();
@@ -139,16 +119,14 @@ private static DynamicMessage convertJsonToProtoMessageImpl(
139119
* @param json
140120
* @param exactJsonKeyName Exact key name in JSONObject instead of lowercased version
141121
* @param currentScope Debugging purposes
142-
* @param allowUnknownFields Ignores unknown JSON fields.
143122
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
144123
*/
145124
private static void fillField(
146125
DynamicMessage.Builder protoMsg,
147126
FieldDescriptor fieldDescriptor,
148127
JSONObject json,
149128
String exactJsonKeyName,
150-
String currentScope,
151-
boolean allowUnknownFields)
129+
String currentScope)
152130
throws IllegalArgumentException {
153131

154132
java.lang.Object val = json.get(exactJsonKeyName);
@@ -204,8 +182,7 @@ private static void fillField(
204182
fieldDescriptor.getMessageType(),
205183
json.getJSONObject(exactJsonKeyName),
206184
currentScope,
207-
/*topLevel =*/ false,
208-
allowUnknownFields));
185+
/*topLevel =*/ false));
209186
return;
210187
}
211188
break;
@@ -224,16 +201,14 @@ private static void fillField(
224201
* @param json If root level has no matching fields, throws exception.
225202
* @param exactJsonKeyName Exact key name in JSONObject instead of lowercased version
226203
* @param currentScope Debugging purposes
227-
* @param allowUnknownFields Ignores unknown JSON fields.
228204
* @throws IllegalArgumentException when JSON data is not compatible with proto descriptor.
229205
*/
230206
private static void fillRepeatedField(
231207
DynamicMessage.Builder protoMsg,
232208
FieldDescriptor fieldDescriptor,
233209
JSONObject json,
234210
String exactJsonKeyName,
235-
String currentScope,
236-
boolean allowUnknownFields)
211+
String currentScope)
237212
throws IllegalArgumentException {
238213

239214
JSONArray jsonArray;
@@ -305,8 +280,7 @@ private static void fillRepeatedField(
305280
fieldDescriptor.getMessageType(),
306281
jsonArray.getJSONObject(i),
307282
currentScope,
308-
/*topLevel =*/ false,
309-
allowUnknownFields));
283+
/*topLevel =*/ false));
310284
} else {
311285
fail = true;
312286
}

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriterTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -876,6 +876,8 @@ public void testFlushAllFailed() throws Exception {
876876
.build();
877877

878878
testBigQueryWrite.addException(Status.DATA_LOSS.asException());
879+
testBigQueryWrite.addException(Status.DATA_LOSS.asException());
880+
testBigQueryWrite.addException(Status.DATA_LOSS.asException());
879881

880882
ApiFuture<AppendRowsResponse> appendFuture1 = sendTestMessage(writer, new String[] {"A"});
881883
ApiFuture<AppendRowsResponse> appendFuture2 = sendTestMessage(writer, new String[] {"B"});

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriterTest.java‎

Lines changed: 17 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -251,8 +251,7 @@ public void testSingleAppendSimpleJson() throws Exception {
251251
AppendRowsResponse.AppendResult.newBuilder().setOffset(Int64Value.of(0)).build())
252252
.build());
253253

254-
ApiFuture<AppendRowsResponse> appendFuture =
255-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
254+
ApiFuture<AppendRowsResponse> appendFuture = writer.append(jsonArr);
256255
assertEquals(0L, appendFuture.get().getAppendResult().getOffset().getValue());
257256
appendFuture.get();
258257
assertEquals(
@@ -299,8 +298,7 @@ public void testSingleAppendMultipleSimpleJson() throws Exception {
299298
AppendRowsResponse.AppendResult.newBuilder().setOffset(Int64Value.of(0)).build())
300299
.build());
301300

302-
ApiFuture<AppendRowsResponse> appendFuture =
303-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
301+
ApiFuture<AppendRowsResponse> appendFuture = writer.append(jsonArr);
304302

305303
assertEquals(0L, appendFuture.get().getAppendResult().getOffset().getValue());
306304
appendFuture.get();
@@ -357,7 +355,7 @@ public void testMultipleAppendSimpleJson() throws Exception {
357355
.build());
358356
ApiFuture<AppendRowsResponse> appendFuture;
359357
for (int i = 0; i < 4; i++) {
360-
appendFuture = writer.append(jsonArr, -1, /* allowUnknownFields */ false);
358+
appendFuture = writer.append(jsonArr);
361359
assertEquals((long) i, appendFuture.get().getAppendResult().getOffset().getValue());
362360
appendFuture.get();
363361
assertEquals(
@@ -443,8 +441,7 @@ public void testSingleAppendComplexJson() throws Exception {
443441
AppendRowsResponse.AppendResult.newBuilder().setOffset(Int64Value.of(0)).build())
444442
.build());
445443

446-
ApiFuture<AppendRowsResponse> appendFuture =
447-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
444+
ApiFuture<AppendRowsResponse> appendFuture = writer.append(jsonArr);
448445

449446
assertEquals(0L, appendFuture.get().getAppendResult().getOffset().getValue());
450447
appendFuture.get();
@@ -495,8 +492,7 @@ public void testAppendMultipleSchemaUpdate() throws Exception {
495492
JSONArray jsonArr = new JSONArray();
496493
jsonArr.put(foo);
497494

498-
ApiFuture<AppendRowsResponse> appendFuture1 =
499-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
495+
ApiFuture<AppendRowsResponse> appendFuture1 = writer.append(jsonArr);
500496

501497
int millis = 0;
502498
while (millis <= 10000) {
@@ -532,8 +528,7 @@ public void testAppendMultipleSchemaUpdate() throws Exception {
532528
JSONArray updatedJsonArr = new JSONArray();
533529
updatedJsonArr.put(updatedFoo);
534530

535-
ApiFuture<AppendRowsResponse> appendFuture2 =
536-
writer.append(updatedJsonArr, -1, /* allowUnknownFields */ false);
531+
ApiFuture<AppendRowsResponse> appendFuture2 = writer.append(updatedJsonArr);
537532

538533
millis = 0;
539534
while (millis <= 10000) {
@@ -570,8 +565,7 @@ public void testAppendMultipleSchemaUpdate() throws Exception {
570565
JSONArray updatedJsonArr2 = new JSONArray();
571566
updatedJsonArr2.put(updatedFoo2);
572567

573-
ApiFuture<AppendRowsResponse> appendFuture3 =
574-
writer.append(updatedJsonArr2, -1, /* allowUnknownFields */ false);
568+
ApiFuture<AppendRowsResponse> appendFuture3 = writer.append(updatedJsonArr2);
575569

576570
assertEquals(2L, appendFuture3.get().getAppendResult().getOffset().getValue());
577571
assertEquals(
@@ -614,8 +608,7 @@ public void testAppendOutOfRangeException() throws Exception {
614608
foo.put("foo", "allen");
615609
JSONArray jsonArr = new JSONArray();
616610
jsonArr.put(foo);
617-
ApiFuture<AppendRowsResponse> appendFuture =
618-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
611+
ApiFuture<AppendRowsResponse> appendFuture = writer.append(jsonArr);
619612
try {
620613
appendFuture.get();
621614
Assert.fail("expected ExecutionException");
@@ -644,8 +637,7 @@ public void testAppendOutOfRangeAndUpdateSchema() throws Exception {
644637
foo.put("foo", "allen");
645638
JSONArray jsonArr = new JSONArray();
646639
jsonArr.put(foo);
647-
ApiFuture<AppendRowsResponse> appendFuture =
648-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
640+
ApiFuture<AppendRowsResponse> appendFuture = writer.append(jsonArr);
649641
try {
650642
appendFuture.get();
651643
Assert.fail("expected ExecutionException");
@@ -668,8 +660,7 @@ public void testAppendOutOfRangeAndUpdateSchema() throws Exception {
668660
JSONArray updatedJsonArr = new JSONArray();
669661
updatedJsonArr.put(updatedFoo);
670662

671-
ApiFuture<AppendRowsResponse> appendFuture2 =
672-
writer.append(updatedJsonArr, -1, /* allowUnknownFields */ false);
663+
ApiFuture<AppendRowsResponse> appendFuture2 = writer.append(updatedJsonArr);
673664
assertEquals(0L, appendFuture2.get().getAppendResult().getOffset().getValue());
674665
appendFuture2.get();
675666
assertEquals(
@@ -727,12 +718,9 @@ public void testSchemaUpdateWithNonemptyBatch() throws Exception {
727718
JSONArray jsonArr = new JSONArray();
728719
jsonArr.put(foo);
729720

730-
ApiFuture<AppendRowsResponse> appendFuture1 =
731-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
732-
ApiFuture<AppendRowsResponse> appendFuture2 =
733-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
734-
ApiFuture<AppendRowsResponse> appendFuture3 =
735-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
721+
ApiFuture<AppendRowsResponse> appendFuture1 = writer.append(jsonArr);
722+
ApiFuture<AppendRowsResponse> appendFuture2 = writer.append(jsonArr);
723+
ApiFuture<AppendRowsResponse> appendFuture3 = writer.append(jsonArr);
736724

737725
assertEquals(0L, appendFuture1.get().getAppendResult().getOffset().getValue());
738726
assertEquals(1L, appendFuture2.get().getAppendResult().getOffset().getValue());
@@ -796,8 +784,7 @@ public void testSchemaUpdateWithNonemptyBatch() throws Exception {
796784
JSONArray updatedJsonArr = new JSONArray();
797785
updatedJsonArr.put(updatedFoo);
798786

799-
ApiFuture<AppendRowsResponse> appendFuture4 =
800-
writer.append(updatedJsonArr, -1, /* allowUnknownFields */ false);
787+
ApiFuture<AppendRowsResponse> appendFuture4 = writer.append(updatedJsonArr);
801788

802789
assertEquals(3L, appendFuture4.get().getAppendResult().getOffset().getValue());
803790
assertEquals(
@@ -857,8 +844,7 @@ public void testMultiThreadAppendNoSchemaUpdate() throws Exception {
857844
new Runnable() {
858845
public void run() {
859846
try {
860-
ApiFuture<AppendRowsResponse> appendFuture =
861-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
847+
ApiFuture<AppendRowsResponse> appendFuture = writer.append(jsonArr);
862848
AppendRowsResponse response = appendFuture.get();
863849
offsetSets.remove(response.getAppendResult().getOffset().getValue());
864850
} catch (Exception e) {
@@ -940,8 +926,7 @@ public void testMultiThreadAppendWithSchemaUpdate() throws Exception {
940926
new Runnable() {
941927
public void run() {
942928
try {
943-
ApiFuture<AppendRowsResponse> appendFuture =
944-
writer.append(jsonArr, -1, /* allowUnknownFields */ false);
929+
ApiFuture<AppendRowsResponse> appendFuture = writer.append(jsonArr);
945930
AppendRowsResponse response = appendFuture.get();
946931
offsetSets.remove(response.getAppendResult().getOffset().getValue());
947932
} catch (Exception e) {
@@ -1004,8 +989,7 @@ public void run() {
1004989
new Runnable() {
1005990
public void run() {
1006991
try {
1007-
ApiFuture<AppendRowsResponse> appendFuture =
1008-
writer.append(jsonArr2, -1, /* allowUnknownFields */ false);
992+
ApiFuture<AppendRowsResponse> appendFuture = writer.append(jsonArr2);
1009993
AppendRowsResponse response = appendFuture.get();
1010994
offsetSets.remove(response.getAppendResult().getOffset().getValue());
1011995
} catch (Exception e) {

0 commit comments

Comments
 (0)