Skip to content
This repository was archived by the owner on Nov 12, 2024. It is now read-only.

Multivalue default support #23

Merged
merged 3 commits into from
Dec 28, 2020
Merged

Multivalue default support #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a3b6533
fix: support empty array default values for mv cols
aaron-steinfeld Dec 28, 2020
b498d31
fix: handle default value for multivalued columns
aaron-steinfeld Dec 28, 2020
148921e
refactor: move logic around for clarity
aaron-steinfeld Dec 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.io.File;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -154,15 +155,18 @@ public static Schema convertAvroSchemaToPinotSchema(org.apache.avro.Schema avroS

for (FieldSpec convertedSpec : fieldSpecs) {
if (convertedSpec != null) {
Object defaultVal;
Object defaultVal = field.defaultVal();
// Replace the avro generated defaults in certain cases
if (field.schema().getType().equals(Type.MAP)) {
// we need to set it empty string instead of the default map because it's being split
// to 2 different string
// A map is split into two multivalued string cols, use an empty string default for each
// TODO - why not use null and let pinot decide?
defaultVal = "";
} else {
defaultVal = field.defaultVal();
}
if (defaultVal == JsonProperties.NULL_VALUE) {
} else if (defaultVal == JsonProperties.NULL_VALUE) {
defaultVal = null;
} else if (!AvroUtils.isSingleValueField(field)
&& defaultVal instanceof Collection
&& ((Collection<?>) defaultVal).isEmpty()) {
// Convert an empty collection into a null for a multivalued col
defaultVal = null;
}
if (defaultVal != null) {
Expand Down
7 changes: 6 additions & 1 deletion ...creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.hypertrace.core.viewcreator.pinot;

import static org.apache.pinot.spi.data.FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING;

import com.typesafe.config.ConfigFactory;
import java.io.File;
import java.util.List;
Expand Down Expand Up @@ -42,7 +44,10 @@ public void testCreatePinotSchemaForView() {
pinotSchemaForView.getDimensionSpec("properties__VALUES").getDataType());
Assertions.assertEquals("",
pinotSchemaForView.getDimensionSpec("properties__KEYS").getDefaultNullValue());

Assertions.assertEquals(DEFAULT_DIMENSION_NULL_VALUE_OF_STRING, pinotSchemaForView.getDimensionSpec("name")
.getDefaultNullValue());
Assertions.assertEquals(DEFAULT_DIMENSION_NULL_VALUE_OF_STRING, pinotSchemaForView.getDimensionSpec("friends")
.getDefaultNullValue());
// metric fields are not part of dimension columns
Assertions.assertEquals("time_taken_millis", pinotSchemaForView.getMetricFieldSpecs().get(0).getName());
Assertions.assertEquals(DataType.LONG, pinotSchemaForView.getMetricFieldSpecs().get(0).getDataType());
Expand Down