Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit 9c8dc0a

Browse files
burakgokburakgok
authored
feat: DATE and DOUBLE support more input formats (#1397)
1. Long values are converted into Integer for DATE fields. 2. Date literals are converted into epoch days (Integer) for DATE fields. 3. Number values are converted into Double for DOUBLE fields.
1 parent 75aed71 commit 9c8dc0a

File tree

3 files changed

+93
-15
lines changed

3 files changed

+93
-15
lines changed

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import com.google.protobuf.Message;
2626
import com.google.protobuf.UninitializedMessageException;
2727
import java.math.BigDecimal;
28+
import java.time.LocalDate;
2829
import java.util.List;
2930
import java.util.logging.Logger;
3031
import org.json.JSONArray;
@@ -269,6 +270,15 @@ private static void fillField(
269270
}
270271
break;
271272
case INT32:
273+
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) {
274+
if (val instanceof String) {
275+
protoMsg.setField(fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay());
276+
return;
277+
} else if (val instanceof Integer || val instanceof Long) {
278+
protoMsg.setField(fieldDescriptor, ((Number) val).intValue());
279+
return;
280+
}
281+
}
272282
if (val instanceof Integer) {
273283
protoMsg.setField(fieldDescriptor, (Integer) val);
274284
return;
@@ -281,11 +291,8 @@ private static void fillField(
281291
}
282292
break;
283293
case DOUBLE:
284-
if (val instanceof Double) {
285-
protoMsg.setField(fieldDescriptor, (Double) val);
286-
return;
287-
} else if (val instanceof Float) {
288-
protoMsg.setField(fieldDescriptor, new Double((Float) val));
294+
if (val instanceof Number) {
295+
protoMsg.setField(fieldDescriptor, ((Number) val).doubleValue());
289296
return;
290297
}
291298
break;
@@ -435,7 +442,16 @@ private static void fillRepeatedField(
435442
}
436443
break;
437444
case INT32:
438-
if (val instanceof Integer) {
445+
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) {
446+
if (val instanceof String) {
447+
protoMsg.addRepeatedField(
448+
fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay());
449+
} else if (val instanceof Integer || val instanceof Long) {
450+
protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).intValue());
451+
} else {
452+
fail = true;
453+
}
454+
} else if (val instanceof Integer) {
439455
protoMsg.addRepeatedField(fieldDescriptor, (Integer) val);
440456
} else {
441457
fail = true;
@@ -449,10 +465,8 @@ private static void fillRepeatedField(
449465
}
450466
break;
451467
case DOUBLE:
452-
if (val instanceof Double) {
453-
protoMsg.addRepeatedField(fieldDescriptor, (Double) val);
454-
} else if (val instanceof Float) {
455-
protoMsg.addRepeatedField(fieldDescriptor, new Double((float) val));
468+
if (val instanceof Number) {
469+
protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).doubleValue());
456470
} else {
457471
fail = true;
458472
}

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java‎

Lines changed: 60 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,11 @@ public class JsonToProtoMessageTest {
7676
new Message[] {Int32Type.newBuilder().setTestFieldType(Integer.MAX_VALUE).build()})
7777
.put(
7878
DoubleType.getDescriptor(),
79-
new Message[] {DoubleType.newBuilder().setTestFieldType(1.23).build()})
79+
new Message[] {
80+
DoubleType.newBuilder().setTestFieldType(Long.MAX_VALUE).build(),
81+
DoubleType.newBuilder().setTestFieldType(Integer.MAX_VALUE).build(),
82+
DoubleType.newBuilder().setTestFieldType(1.23).build()
83+
})
8084
.put(
8185
StringType.getDescriptor(),
8286
new Message[] {StringType.newBuilder().setTestFieldType("test").build()})
@@ -181,6 +185,26 @@ public class JsonToProtoMessageTest {
181185
.put(
182186
RepeatedDouble.getDescriptor(),
183187
new Message[] {
188+
RepeatedDouble.newBuilder()
189+
.addTestRepeated(Long.MAX_VALUE)
190+
.addTestRepeated(Long.MIN_VALUE)
191+
.addTestRepeated(Integer.MAX_VALUE)
192+
.addTestRepeated(Integer.MIN_VALUE)
193+
.addTestRepeated(Short.MAX_VALUE)
194+
.addTestRepeated(Short.MIN_VALUE)
195+
.addTestRepeated(Byte.MAX_VALUE)
196+
.addTestRepeated(Byte.MIN_VALUE)
197+
.addTestRepeated(0)
198+
.build(),
199+
RepeatedDouble.newBuilder()
200+
.addTestRepeated(Integer.MAX_VALUE)
201+
.addTestRepeated(Integer.MIN_VALUE)
202+
.addTestRepeated(Short.MAX_VALUE)
203+
.addTestRepeated(Short.MIN_VALUE)
204+
.addTestRepeated(Byte.MAX_VALUE)
205+
.addTestRepeated(Byte.MIN_VALUE)
206+
.addTestRepeated(0)
207+
.build(),
184208
RepeatedDouble.newBuilder()
185209
.addTestRepeated(Double.MAX_VALUE)
186210
.addTestRepeated(Double.MIN_VALUE)
@@ -593,15 +617,43 @@ public void testBigNumericMismatch() throws Exception {
593617

594618
@Test
595619
public void testDouble() throws Exception {
596-
TestDouble expectedProto = TestDouble.newBuilder().setDouble(1.2).setFloat(3.4f).build();
620+
TestDouble expectedProto =
621+
TestDouble.newBuilder()
622+
.setDouble(1.2)
623+
.setFloat(3.4f)
624+
.setByte(5)
625+
.setShort(6)
626+
.setInt(7)
627+
.setLong(8)
628+
.build();
597629
JSONObject json = new JSONObject();
598630
json.put("double", 1.2);
599631
json.put("float", 3.4f);
632+
json.put("byte", new Byte((byte) 5));
633+
json.put("short", new Short((short) 6));
634+
json.put("int", 7);
635+
json.put("long", 8L);
600636
DynamicMessage protoMsg =
601637
JsonToProtoMessage.convertJsonToProtoMessage(TestDouble.getDescriptor(), json);
602638
assertEquals(expectedProto, protoMsg);
603639
}
604640

641+
@Test
642+
public void testDate() throws Exception {
643+
TableSchema tableSchema =
644+
TableSchema.newBuilder()
645+
.addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_string").build())
646+
.addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_long").build())
647+
.build();
648+
TestDate expectedProto = TestDate.newBuilder().setTestString(18935).setTestLong(18935).build();
649+
JSONObject json = new JSONObject();
650+
json.put("test_string", "2021-11-04");
651+
json.put("test_long", 18935L);
652+
DynamicMessage protoMsg =
653+
JsonToProtoMessage.convertJsonToProtoMessage(TestDate.getDescriptor(), tableSchema, json);
654+
assertEquals(expectedProto, protoMsg);
655+
}
656+
605657
@Test
606658
public void testAllTypes() throws Exception {
607659
for (Map.Entry<Descriptor, String> entry : AllTypesToDebugMessageTest.entrySet()) {
@@ -620,7 +672,9 @@ public void testAllTypes() throws Exception {
620672
e.getMessage());
621673
}
622674
}
623-
if (entry.getKey() == Int64Type.getDescriptor()
675+
if (entry.getKey() == DoubleType.getDescriptor()) {
676+
assertEquals(entry.getKey().getFullName(), 3, success);
677+
} else if (entry.getKey() == Int64Type.getDescriptor()
624678
|| entry.getKey() == BytesType.getDescriptor()) {
625679
assertEquals(entry.getKey().getFullName(), 2, success);
626680
} else {
@@ -656,8 +710,9 @@ public void testAllRepeatedTypesWithLimits() throws Exception {
656710
.equals("Error: root.test_repeated[0] could not be converted to byte[]."));
657711
}
658712
}
659-
if (entry.getKey() == RepeatedInt64.getDescriptor()
660-
|| entry.getKey() == RepeatedDouble.getDescriptor()) {
713+
if (entry.getKey() == RepeatedDouble.getDescriptor()) {
714+
assertEquals(entry.getKey().getFullName(), 4, success);
715+
} else if (entry.getKey() == RepeatedInt64.getDescriptor()) {
661716
assertEquals(entry.getKey().getFullName(), 2, success);
662717
} else {
663718
assertEquals(entry.getKey().getFullName(), 1, success);

‎google-cloud-bigquerystorage/src/test/proto/jsonTest.proto‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,15 @@ message TestInt32 {
119119
message TestDouble {
120120
optional double double = 1;
121121
optional double float = 2;
122+
optional double byte = 3;
123+
optional double short = 4;
124+
optional double int = 5;
125+
optional double long = 6;
126+
}
127+
128+
message TestDate {
129+
optional int32 test_string = 1;
130+
optional int32 test_long = 2;
122131
}
123132

124133
message NestedRepeated {

0 commit comments

Comments
 (0)