diff --git a/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java b/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java index 0aff7f8..11f001c 100644 --- a/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java +++ b/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.FileWriter; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -154,15 +155,18 @@ public static Schema convertAvroSchemaToPinotSchema(org.apache.avro.Schema avroS for (FieldSpec convertedSpec : fieldSpecs) { if (convertedSpec != null) { - Object defaultVal; + Object defaultVal = field.defaultVal(); + // Replace the avro generated defaults in certain cases if (field.schema().getType().equals(Type.MAP)) { - // we need to set it empty string instead of the default map because it's being split - // to 2 different string + // A map is split into two multivalued string cols, use an empty string default for each + // TODO - why not use null and let pinot decide? defaultVal = ""; - } else { - defaultVal = field.defaultVal(); - } - if (defaultVal == JsonProperties.NULL_VALUE) { + } else if (defaultVal == JsonProperties.NULL_VALUE) { + defaultVal = null; + } else if (!AvroUtils.isSingleValueField(field) + && defaultVal instanceof Collection + && ((Collection) defaultVal).isEmpty()) { + // Convert an empty collection into a null for a multivalued col defaultVal = null; } if (defaultVal != null) { diff --git a/view-creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java b/view-creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java index f535b97..5ac8ff6 100644 --- a/view-creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java +++ b/view-creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java @@ -1,5 +1,7 @@ package org.hypertrace.core.viewcreator.pinot; +import static org.apache.pinot.spi.data.FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING; + import com.typesafe.config.ConfigFactory; import java.io.File; import java.util.List; @@ -42,7 +44,10 @@ public void testCreatePinotSchemaForView() { pinotSchemaForView.getDimensionSpec("properties__VALUES").getDataType()); Assertions.assertEquals("", pinotSchemaForView.getDimensionSpec("properties__KEYS").getDefaultNullValue()); - + Assertions.assertEquals(DEFAULT_DIMENSION_NULL_VALUE_OF_STRING, pinotSchemaForView.getDimensionSpec("name") + .getDefaultNullValue()); + Assertions.assertEquals(DEFAULT_DIMENSION_NULL_VALUE_OF_STRING, pinotSchemaForView.getDimensionSpec("friends") + .getDefaultNullValue()); // metric fields are not part of dimension columns Assertions.assertEquals("time_taken_millis", pinotSchemaForView.getMetricFieldSpecs().get(0).getName()); Assertions.assertEquals(DataType.LONG, pinotSchemaForView.getMetricFieldSpecs().get(0).getDataType());