Skip to content
Permalink
Browse files
feat: DATE and DOUBLE support more input formats (#1397)
1. Long values are converted into Integer for DATE fields.
2. Date literals are converted into epoch days (Integer) for DATE fields.
3. Number values are converted into Double for DOUBLE fields.
  • Loading branch information
burakgok committed Nov 9, 2021
1 parent 75aed71 commit 9c8dc0aabb471fdb8580f434cc5a66bad585e8f2
@@ -25,6 +25,7 @@
import com.google.protobuf.Message;
import com.google.protobuf.UninitializedMessageException;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.util.List;
import java.util.logging.Logger;
import org.json.JSONArray;
@@ -269,6 +270,15 @@ private static void fillField(
}
break;
case INT32:
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) {
if (val instanceof String) {
protoMsg.setField(fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay());
return;
} else if (val instanceof Integer || val instanceof Long) {
protoMsg.setField(fieldDescriptor, ((Number) val).intValue());
return;
}
}
if (val instanceof Integer) {
protoMsg.setField(fieldDescriptor, (Integer) val);
return;
@@ -281,11 +291,8 @@ private static void fillField(
}
break;
case DOUBLE:
if (val instanceof Double) {
protoMsg.setField(fieldDescriptor, (Double) val);
return;
} else if (val instanceof Float) {
protoMsg.setField(fieldDescriptor, new Double((Float) val));
if (val instanceof Number) {
protoMsg.setField(fieldDescriptor, ((Number) val).doubleValue());
return;
}
break;
@@ -435,7 +442,16 @@ private static void fillRepeatedField(
}
break;
case INT32:
if (val instanceof Integer) {
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) {
if (val instanceof String) {
protoMsg.addRepeatedField(
fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay());
} else if (val instanceof Integer || val instanceof Long) {
protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).intValue());
} else {
fail = true;
}
} else if (val instanceof Integer) {
protoMsg.addRepeatedField(fieldDescriptor, (Integer) val);
} else {
fail = true;
@@ -449,10 +465,8 @@ private static void fillRepeatedField(
}
break;
case DOUBLE:
if (val instanceof Double) {
protoMsg.addRepeatedField(fieldDescriptor, (Double) val);
} else if (val instanceof Float) {
protoMsg.addRepeatedField(fieldDescriptor, new Double((float) val));
if (val instanceof Number) {
protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).doubleValue());
} else {
fail = true;
}
@@ -76,7 +76,11 @@ public class JsonToProtoMessageTest {
new Message[] {Int32Type.newBuilder().setTestFieldType(Integer.MAX_VALUE).build()})
.put(
DoubleType.getDescriptor(),
new Message[] {DoubleType.newBuilder().setTestFieldType(1.23).build()})
new Message[] {
DoubleType.newBuilder().setTestFieldType(Long.MAX_VALUE).build(),
DoubleType.newBuilder().setTestFieldType(Integer.MAX_VALUE).build(),
DoubleType.newBuilder().setTestFieldType(1.23).build()
})
.put(
StringType.getDescriptor(),
new Message[] {StringType.newBuilder().setTestFieldType("test").build()})
@@ -181,6 +185,26 @@ public class JsonToProtoMessageTest {
.put(
RepeatedDouble.getDescriptor(),
new Message[] {
RepeatedDouble.newBuilder()
.addTestRepeated(Long.MAX_VALUE)
.addTestRepeated(Long.MIN_VALUE)
.addTestRepeated(Integer.MAX_VALUE)
.addTestRepeated(Integer.MIN_VALUE)
.addTestRepeated(Short.MAX_VALUE)
.addTestRepeated(Short.MIN_VALUE)
.addTestRepeated(Byte.MAX_VALUE)
.addTestRepeated(Byte.MIN_VALUE)
.addTestRepeated(0)
.build(),
RepeatedDouble.newBuilder()
.addTestRepeated(Integer.MAX_VALUE)
.addTestRepeated(Integer.MIN_VALUE)
.addTestRepeated(Short.MAX_VALUE)
.addTestRepeated(Short.MIN_VALUE)
.addTestRepeated(Byte.MAX_VALUE)
.addTestRepeated(Byte.MIN_VALUE)
.addTestRepeated(0)
.build(),
RepeatedDouble.newBuilder()
.addTestRepeated(Double.MAX_VALUE)
.addTestRepeated(Double.MIN_VALUE)
@@ -593,15 +617,43 @@ public void testBigNumericMismatch() throws Exception {

@Test
public void testDouble() throws Exception {
TestDouble expectedProto = TestDouble.newBuilder().setDouble(1.2).setFloat(3.4f).build();
TestDouble expectedProto =
TestDouble.newBuilder()
.setDouble(1.2)
.setFloat(3.4f)
.setByte(5)
.setShort(6)
.setInt(7)
.setLong(8)
.build();
JSONObject json = new JSONObject();
json.put("double", 1.2);
json.put("float", 3.4f);
json.put("byte", new Byte((byte) 5));
json.put("short", new Short((short) 6));
json.put("int", 7);
json.put("long", 8L);
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(TestDouble.getDescriptor(), json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testDate() throws Exception {
TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_string").build())
.addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_long").build())
.build();
TestDate expectedProto = TestDate.newBuilder().setTestString(18935).setTestLong(18935).build();
JSONObject json = new JSONObject();
json.put("test_string", "2021-11-04");
json.put("test_long", 18935L);
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(TestDate.getDescriptor(), tableSchema, json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testAllTypes() throws Exception {
for (Map.Entry<Descriptor, String> entry : AllTypesToDebugMessageTest.entrySet()) {
@@ -620,7 +672,9 @@ public void testAllTypes() throws Exception {
e.getMessage());
}
}
if (entry.getKey() == Int64Type.getDescriptor()
if (entry.getKey() == DoubleType.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 3, success);
} else if (entry.getKey() == Int64Type.getDescriptor()
|| entry.getKey() == BytesType.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 2, success);
} else {
@@ -656,8 +710,9 @@ public void testAllRepeatedTypesWithLimits() throws Exception {
.equals("Error: root.test_repeated[0] could not be converted to byte[]."));
}
}
if (entry.getKey() == RepeatedInt64.getDescriptor()
|| entry.getKey() == RepeatedDouble.getDescriptor()) {
if (entry.getKey() == RepeatedDouble.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 4, success);
} else if (entry.getKey() == RepeatedInt64.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 2, success);
} else {
assertEquals(entry.getKey().getFullName(), 1, success);
@@ -119,6 +119,15 @@ message TestInt32 {
message TestDouble {
optional double double = 1;
optional double float = 2;
optional double byte = 3;
optional double short = 4;
optional double int = 5;
optional double long = 6;
}

message TestDate {
optional int32 test_string = 1;
optional int32 test_long = 2;
}

message NestedRepeated {

0 comments on commit 9c8dc0a

Please sign in to comment.