Skip to content

Commit

Permalink
Change DictionaryBlock compact check to be lazy
Browse files Browse the repository at this point in the history
During construction of a DictionaryBlock the compact size is calculted by
getting the length of the used dictionary positions, but this information
is only needed in some cases.
  • Loading branch information
dain committed Dec 15, 2016
1 parent 6155b1d commit 3e48a1b
Showing 1 changed file with 30 additions and 19 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ public class DictionaryBlock
private final Block dictionary;
private final Slice ids;
private final int retainedSizeInBytes;
private final int sizeInBytes;
private final int uniqueIds;
private volatile int sizeInBytes = -1;
private volatile int uniqueIds = -1;
private final DictionaryId dictionarySourceId;

public DictionaryBlock(int positionCount, Block dictionary, Slice ids)
Expand Down Expand Up @@ -81,23 +81,6 @@ public DictionaryBlock(int positionCount, Block dictionary, Slice ids, boolean d
this.sizeInBytes = this.retainedSizeInBytes;
this.uniqueIds = dictionary.getPositionCount();
}
else {
int sizeInBytes = 0;
int uniqueIds = 0;
boolean[] seen = new boolean[dictionary.getPositionCount()];
for (int i = 0; i < positionCount; i++) {
int position = getIndex(ids, i);
if (!seen[position]) {
if (!dictionary.isNull(position)) {
sizeInBytes += dictionary.getLength(position);
}
uniqueIds++;
seen[position] = true;
}
}
this.sizeInBytes = sizeInBytes + ids.length();
this.uniqueIds = uniqueIds;
}
}

@Override
Expand Down Expand Up @@ -199,9 +182,33 @@ public int getPositionCount()
@Override
public int getSizeInBytes()
{
// this is racy but is safe because sizeInBytes is an int and the calculation is stable
if (sizeInBytes < 0) {
calculateCompactSize();
}
return sizeInBytes;
}

private void calculateCompactSize()
{
int sizeInBytes = 0;
int uniqueIds = 0;
boolean[] seen = new boolean[dictionary.getPositionCount()];
for (int i = 0; i < positionCount; i++) {
int position = getIndex(ids, i);
if (!seen[position]) {
if (!dictionary.isNull(position)) {
// todo this is wrong for ArrayBlock and InterleavedBlock as length means entry count
sizeInBytes += dictionary.getLength(position);
}
uniqueIds++;
seen[position] = true;
}
}
this.sizeInBytes = sizeInBytes;
this.uniqueIds = uniqueIds;
}

@Override
public int getRetainedSizeInBytes()
{
Expand Down Expand Up @@ -283,6 +290,10 @@ public DictionaryId getDictionarySourceId()

public boolean isCompact()
{
// this is racy but is safe because sizeInBytes is an int and the calculation is stable
if (uniqueIds < 0) {
calculateCompactSize();
}
return uniqueIds == dictionary.getPositionCount();
}

Expand Down

0 comments on commit 3e48a1b

Please sign in to comment.