Skip to content
Permalink
Browse files
Fix indexMerger to respect the includeAllDimensions flag (#12428)
* Fix indexMerger to respect flag includeAllDimensions flag; jsonInputFormat should set keepNullColumns if useFieldDiscovery is set

* address comments
  • Loading branch information
jihoonson committed Apr 13, 2022
1 parent f24e9c6 commit 5e5625f3ae9fa663697214f49e66b5c296d342fb
Showing 6 changed files with 517 additions and 341 deletions.
@@ -74,7 +74,11 @@ public JsonInputFormat(
super(flattenSpec);
this.featureSpec = featureSpec == null ? Collections.emptyMap() : featureSpec;
this.objectMapper = new ObjectMapper();
this.keepNullColumns = keepNullColumns == null ? false : keepNullColumns;
if (keepNullColumns != null) {
this.keepNullColumns = keepNullColumns;
} else {
this.keepNullColumns = flattenSpec != null && flattenSpec.isUseFieldDiscovery();
}
for (Entry<String, Boolean> entry : this.featureSpec.entrySet()) {
Feature feature = Feature.valueOf(entry.getKey());
objectMapper.configure(feature, entry.getValue());
@@ -88,6 +92,12 @@ public Map<String, Boolean> getFeatureSpec()
return featureSpec;
}

@JsonProperty
public boolean isKeepNullColumns()
{
return keepNullColumns;
}

@Override
public boolean isSplittable()
{
@@ -41,7 +41,7 @@ public void testSerde() throws IOException
final ObjectMapper mapper = new ObjectMapper();
final JsonInputFormat format = new JsonInputFormat(
new JSONPathSpec(
false,
true,
ImmutableList.of(
new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz", "baz"),
new JSONPathFieldSpec(JSONPathFieldType.ROOT, "root_baz2", "baz2"),
@@ -52,7 +52,7 @@ public void testSerde() throws IOException
)
),
ImmutableMap.of(Feature.ALLOW_COMMENTS.name(), true, Feature.ALLOW_UNQUOTED_FIELD_NAMES.name(), false),
false
true
);
final byte[] bytes = mapper.writeValueAsBytes(format);
final JsonInputFormat fromJson = (JsonInputFormat) mapper.readValue(bytes, InputFormat.class);
@@ -72,4 +72,37 @@ public void testEquals()
.withIgnoredFields("objectMapper")
.verify();
}

@Test
public void test_unsetUseFieldDiscovery_unsetKeepNullColumnsByDefault()
{
final JsonInputFormat format = new JsonInputFormat(
new JSONPathSpec(false, null),
null,
null
);
Assert.assertFalse(format.isKeepNullColumns());
}

@Test
public void testUseFieldDiscovery_setKeepNullColumnsByDefault()
{
final JsonInputFormat format = new JsonInputFormat(
new JSONPathSpec(true, null),
null,
null
);
Assert.assertTrue(format.isKeepNullColumns());
}

@Test
public void testUseFieldDiscovery_doNotChangeKeepNullColumnsUserSets()
{
final JsonInputFormat format = new JsonInputFormat(
new JSONPathSpec(true, null),
null,
false
);
Assert.assertFalse(format.isKeepNullColumns());
}
}

This file was deleted.

0 comments on commit 5e5625f

Please sign in to comment.