Skip to content

avoid binary searches in distinct count #7802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
richardstartin wants to merge 1 commit into from
Closed

avoid binary searches in distinct count #7802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions ...va/org/apache/pinot/core/query/aggregation/function/DistinctCountAggregationFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.pinot.spi.utils.ByteArray;
import org.roaringbitmap.PeekableIntIterator;
import org.roaringbitmap.RoaringBitmap;
import org.roaringbitmap.RoaringBitmapWriter;


@SuppressWarnings({"rawtypes", "unchecked"})
Expand Down Expand Up @@ -71,7 +72,10 @@ public void aggregate(int length, AggregationResultHolder aggregationResultHolde
Dictionary dictionary = blockValSet.getDictionary();
if (dictionary != null) {
int[] dictIds = blockValSet.getDictionaryIdsSV();
getDictIdBitmap(aggregationResultHolder, dictionary).addN(dictIds, 0, length);
RoaringBitmapWriter<RoaringBitmap> writer = getDictIdBitmap(aggregationResultHolder, dictionary);
for (int i = 0; i < length; i++) {
writer.add(dictIds[i]);
}
return;
}

Expand Down Expand Up @@ -312,7 +316,7 @@ public Integer extractFinalResult(Set intermediateResult) {
/**
* Returns the dictionary id bitmap from the result holder or creates a new one if it does not exist.
*/
protected static RoaringBitmap getDictIdBitmap(AggregationResultHolder aggregationResultHolder,
protected static RoaringBitmapWriter<RoaringBitmap> getDictIdBitmap(AggregationResultHolder aggregationResultHolder,
Dictionary dictionary) {
DictIdsWrapper dictIdsWrapper = aggregationResultHolder.getResult();
if (dictIdsWrapper == null) {
Expand Down Expand Up @@ -358,8 +362,8 @@ private static Set getValueSet(DataType valueType) {
/**
* Returns the dictionary id bitmap for the given group key or creates a new one if it does not exist.
*/
protected static RoaringBitmap getDictIdBitmap(GroupByResultHolder groupByResultHolder, int groupKey,
Dictionary dictionary) {
protected static RoaringBitmapWriter<RoaringBitmap> getDictIdBitmap(GroupByResultHolder groupByResultHolder,
int groupKey, Dictionary dictionary) {
DictIdsWrapper dictIdsWrapper = groupByResultHolder.getResult(groupKey);
if (dictIdsWrapper == null) {
dictIdsWrapper = new DictIdsWrapper(dictionary);
Expand Down Expand Up @@ -449,7 +453,7 @@ private static void setValueForGroupKeys(GroupByResultHolder groupByResultHolder
*/
private static Set convertToValueSet(DictIdsWrapper dictIdsWrapper) {
Dictionary dictionary = dictIdsWrapper._dictionary;
RoaringBitmap dictIdBitmap = dictIdsWrapper._dictIdBitmap;
RoaringBitmap dictIdBitmap = dictIdsWrapper._dictIdBitmap.get();
int numValues = dictIdBitmap.getCardinality();
PeekableIntIterator iterator = dictIdBitmap.getIntIterator();
DataType storedType = dictionary.getValueType();
Expand Down Expand Up @@ -497,11 +501,11 @@ private static Set convertToValueSet(DictIdsWrapper dictIdsWrapper) {

private static final class DictIdsWrapper {
final Dictionary _dictionary;
final RoaringBitmap _dictIdBitmap;
final RoaringBitmapWriter<RoaringBitmap> _dictIdBitmap;

private DictIdsWrapper(Dictionary dictionary) {
_dictionary = dictionary;
_dictIdBitmap = new RoaringBitmap();
_dictIdBitmap = RoaringBitmapWriter.writer().constantMemory().get();
}
}
}
9 changes: 5 additions & 4 deletions .../org/apache/pinot/core/query/aggregation/function/DistinctCountMVAggregationFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.roaringbitmap.RoaringBitmap;
import org.roaringbitmap.RoaringBitmapWriter;


@SuppressWarnings({"rawtypes", "unchecked"})
Expand All @@ -55,10 +56,10 @@ public void aggregate(int length, AggregationResultHolder aggregationResultHolde
// For dictionary-encoded expression, store dictionary ids into the bitmap
Dictionary dictionary = blockValSet.getDictionary();
if (dictionary != null) {
RoaringBitmap dictIdBitmap = getDictIdBitmap(aggregationResultHolder, dictionary);
RoaringBitmapWriter<RoaringBitmap> dictIdBitmap = getDictIdBitmap(aggregationResultHolder, dictionary);
int[][] dictIds = blockValSet.getDictionaryIdsMV();
for (int i = 0; i < length; i++) {
dictIdBitmap.add(dictIds[i]);
dictIdBitmap.addMany(dictIds[i]);
}
return;
}
Expand Down Expand Up @@ -129,7 +130,7 @@ public void aggregateGroupBySV(int length, int[] groupKeyArray, GroupByResultHol
if (dictionary != null) {
int[][] dictIds = blockValSet.getDictionaryIdsMV();
for (int i = 0; i < length; i++) {
getDictIdBitmap(groupByResultHolder, groupKeyArray[i], dictionary).add(dictIds[i]);
getDictIdBitmap(groupByResultHolder, groupKeyArray[i], dictionary).addMany(dictIds[i]);
}
return;
}
Expand Down Expand Up @@ -203,7 +204,7 @@ public void aggregateGroupByMV(int length, int[][] groupKeysArray, GroupByResult
int[][] dictIds = blockValSet.getDictionaryIdsMV();
for (int i = 0; i < length; i++) {
for (int groupKey : groupKeysArray[i]) {
getDictIdBitmap(groupByResultHolder, groupKey, dictionary).add(dictIds[i]);
getDictIdBitmap(groupByResultHolder, groupKey, dictionary).addMany(dictIds[i]);
}
}
return;
Expand Down