Skip to content

null handling for doubles sketch and array of doubles sketch aggs #9112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jon-wei merged 2 commits into from
Jan 7, 2020
Merged

null handling for doubles sketch and array of doubles sketch aggs #9112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2891576
doubles sketch and array of doubles sketch aggs now skip rows with nu…
clintropolis Dec 31, 2019
cccc149
formatting
clintropolis Dec 31, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ...g/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildAggregator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ public DoublesSketchBuildAggregator(final ColumnValueSelector<Double> valueSelec
@Override
public synchronized void aggregate()
{
if (valueSelector.isNull()) {
return;
}
sketch.update(valueSelector.getDouble());
}

Expand Down
3 changes: 3 additions & 0 deletions ...he/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ public synchronized void init(final ByteBuffer buffer, final int position)
@Override
public synchronized void aggregate(final ByteBuffer buffer, final int position)
{
if (selector.isNull()) {
return;
}
final UpdateDoublesSketch sketch = sketches.get(buffer).get(position);
sketch.update(selector.getDouble());
}
Expand Down
6 changes: 5 additions & 1 deletion ...pache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchBuildAggregator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,11 @@ public void aggregate()
{
final IndexedInts keys = keySelector.getRow();
for (int i = 0; i < valueSelectors.length; i++) {
values[i] = valueSelectors[i].getDouble();
if (valueSelectors[i].isNull()) {
return;
} else {
values[i] = valueSelectors[i].getDouble();
}
}
synchronized (this) {
for (int i = 0, keysSize = keys.size(); i < keysSize; i++) {
Expand Down
6 changes: 5 additions & 1 deletion ...druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchBuildBufferAggregator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,11 @@ public void init(final ByteBuffer buf, final int position)
public void aggregate(final ByteBuffer buf, final int position)
{
for (int i = 0; i < valueSelectors.length; i++) {
values[i] = valueSelectors[i].getDouble();
if (valueSelectors[i].isNull()) {
return;
} else {
values[i] = valueSelectors[i].getDouble();
}
}
final IndexedInts keys = keySelector.getRow();
// Wrapping memory and ArrayOfDoublesSketch is inexpensive compared to sketch operations.
Expand Down
86 changes: 73 additions & 13 deletions ...rg/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.druid.query.aggregation.datasketches.quantiles;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.guava.Sequence;
Expand All @@ -28,6 +29,7 @@
import org.apache.druid.query.groupby.GroupByQueryConfig;
import org.apache.druid.query.groupby.GroupByQueryRunnerTest;
import org.apache.druid.query.groupby.ResultRow;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.After;
import org.junit.Assert;
import org.junit.Rule;
Expand All @@ -43,7 +45,7 @@
import java.util.List;

@RunWith(Parameterized.class)
public class DoublesSketchAggregatorTest
public class DoublesSketchAggregatorTest extends InitializedNullHandlingTest
{

private final AggregationTestHelper helper;
Expand Down Expand Up @@ -211,11 +213,12 @@ public void buildingSketchesAtIngestionTime() throws Exception
" \"dimensionExclusions\": [ \"sequenceNumber\"],",
" \"spatialDimensions\": []",
" },",
" \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]",
" \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\", \"valueWithNulls\"]",
" }",
"}"
),
"[{\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}]",
"[{\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128},"
+ "{\"type\": \"quantilesDoublesSketch\", \"name\": \"sketchWithNulls\", \"fieldName\": \"valueWithNulls\", \"k\": 128}]",
0, // minTimestamp
Granularities.NONE,
10, // maxRowCount
Expand All @@ -228,11 +231,14 @@ public void buildingSketchesAtIngestionTime() throws Exception
" \"dimensions\": [],",
" \"aggregations\": [",
" {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"k\": 128},",
" {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketchWithNulls\", \"fieldName\": \"sketchWithNulls\", \"k\": 128},",
" {\"type\": \"quantilesDoublesSketch\", \"name\": \"non_existent_sketch\", \"fieldName\": \"non_existent_sketch\", \"k\": 128}",
" ],",
" \"postAggregations\": [",
" {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},",
" {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}",
" {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},",
" {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantilesWithNulls\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketchWithNulls\"}},",
" {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogramWithNulls\", \"splitPoints\": [6.25, 7.5, 8.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketchWithNulls\"}}",
" ],",
" \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]",
"}"
Expand All @@ -247,22 +253,44 @@ public void buildingSketchesAtIngestionTime() throws Exception
long sketchValue = (long) sketchObject;
Assert.assertEquals(400, sketchValue);

Object sketchObjectWithNulls = row.get(1);
Assert.assertTrue(sketchObjectWithNulls instanceof Long);
long sketchValueWithNulls = (long) sketchObjectWithNulls;
Assert.assertEquals(377, sketchValueWithNulls);

// post agg
Object quantilesObject = row.get(2);
Object quantilesObject = row.get(3);
Assert.assertTrue(quantilesObject instanceof double[]);
double[] quantiles = (double[]) quantilesObject;
Assert.assertEquals(0, quantiles[0], 0.05); // min value
Assert.assertEquals(0.5, quantiles[1], 0.05); // median value
Assert.assertEquals(1, quantiles[2], 0.05); // max value

// post agg
Object histogramObject = row.get(3);
Object histogramObject = row.get(4);
Assert.assertTrue(histogramObject instanceof double[]);
double[] histogram = (double[]) histogramObject;
Assert.assertEquals(4, histogram.length);
for (final double bin : histogram) {
Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly distributed into 4 bins
}

// post agg with nulls
Object quantilesObjectWithNulls = row.get(5);
Assert.assertTrue(quantilesObjectWithNulls instanceof double[]);
double[] quantilesWithNulls = (double[]) quantilesObjectWithNulls;
Assert.assertEquals(5.0, quantilesWithNulls[0], 0.05); // min value
Assert.assertEquals(7.55, quantilesWithNulls[1], 0.05); // median value
Assert.assertEquals(10.0, quantilesWithNulls[2], 0.05); // max value

// post agg with nulls
Object histogramObjectWithNulls = row.get(6);
Assert.assertTrue(histogramObjectWithNulls instanceof double[]);
double[] histogramWithNulls = (double[]) histogramObjectWithNulls;
Assert.assertEquals(4, histogramWithNulls.length);
for (final double bin : histogramWithNulls) {
Assert.assertEquals(100, bin, 50); // distribution is skewed due to nulls
}
}

@Test
Expand All @@ -282,11 +310,12 @@ public void buildingSketchesAtQueryTime() throws Exception
" \"dimensionExclusions\": [],",
" \"spatialDimensions\": []",
" },",
" \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]",
" \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\", \"valueWithNulls\"]",
" }",
"}"
),
"[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]",
"[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"},"
+ "{\"type\": \"doubleSum\", \"name\": \"valueWithNulls\", \"fieldName\": \"valueWithNulls\"}]",
0, // minTimestamp
Granularities.NONE,
10, // maxRowCount
Expand All @@ -298,12 +327,16 @@ public void buildingSketchesAtQueryTime() throws Exception
" \"granularity\": \"ALL\",",
" \"dimensions\": [],",
" \"aggregations\": [",
" {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}",
" {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128},",
" {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketchWithNulls\", \"fieldName\": \"valueWithNulls\", \"k\": 128}",
" ],",
" \"postAggregations\": [",
" {\"type\": \"quantilesDoublesSketchToQuantile\", \"name\": \"quantile\", \"fraction\": 0.5, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},",
" {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},",
" {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}",
" {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},",
" {\"type\": \"quantilesDoublesSketchToQuantile\", \"name\": \"quantileWithNulls\", \"fraction\": 0.5, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketchWithNulls\"}},",
" {\"type\": \"quantilesDoublesSketchToQuantiles\", \"name\": \"quantilesWithNulls\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketchWithNulls\"}},",
" {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogramWithNulls\", \"splitPoints\": [6.25, 7.5, 8.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketchWithNulls\"}}",
" ],",
" \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]",
"}"
Expand All @@ -318,27 +351,54 @@ public void buildingSketchesAtQueryTime() throws Exception
long sketchValue = (long) sketchObject;
Assert.assertEquals(400, sketchValue);

Object sketchObjectWithNulls = row.get(1);
Assert.assertTrue(sketchObjectWithNulls instanceof Long);
long sketchValueWithNulls = (long) sketchObjectWithNulls;
Assert.assertEquals(NullHandling.replaceWithDefault() ? 400 : 377, sketchValueWithNulls);

// post agg
Object quantileObject = row.get(1);
Object quantileObject = row.get(2);
Assert.assertTrue(quantileObject instanceof Double);
Assert.assertEquals(0.5, (double) quantileObject, 0.05); // median value

// post agg
Object quantilesObject = row.get(2);
Object quantilesObject = row.get(3);
Assert.assertTrue(quantilesObject instanceof double[]);
double[] quantiles = (double[]) quantilesObject;
Assert.assertEquals(0, quantiles[0], 0.05); // min value
Assert.assertEquals(0.5, quantiles[1], 0.05); // median value
Assert.assertEquals(1, quantiles[2], 0.05); // max value

// post agg
Object histogramObject = row.get(3);
Object histogramObject = row.get(4);
Assert.assertTrue(histogramObject instanceof double[]);
double[] histogram = (double[]) histogramObject;
for (final double bin : histogram) {
Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly
// distributed into 4 bins
}

// post agg with nulls
Object quantileObjectWithNulls = row.get(5);
Assert.assertTrue(quantileObjectWithNulls instanceof Double);
Assert.assertEquals(NullHandling.replaceWithDefault() ? 7.4 : 7.5, (double) quantileObjectWithNulls, 0.1); // median value

// post agg with nulls
Object quantilesObjectWithNulls = row.get(6);
Assert.assertTrue(quantilesObjectWithNulls instanceof double[]);
double[] quantilesWithNulls = (double[]) quantilesObjectWithNulls;
Assert.assertEquals(NullHandling.replaceWithDefault() ? 0.0 : 5.0, quantilesWithNulls[0], 0.05); // min value
Assert.assertEquals(NullHandling.replaceWithDefault() ? 7.4 : 7.5, quantilesWithNulls[1], 0.1); // median value
Assert.assertEquals(10.0, quantilesWithNulls[2], 0.05); // max value

// post agg with nulls
Object histogramObjectWithNulls = row.get(7);
Assert.assertTrue(histogramObjectWithNulls instanceof double[]);
double[] histogramWithNulls = (double[]) histogramObjectWithNulls;
for (final double bin : histogramWithNulls) {
Assert.assertEquals(100, bin, 80); // distribution is skewed due to nulls/0s
// distributed into 4 bins
}
}

@Test
Expand Down
Loading