Skip to content

Commit

Permalink
Use buffered bytes estimation from writer in DictionaryColumnManager
Browse files Browse the repository at this point in the history
  • Loading branch information
@wenleix
wenleix committed Sep 3, 2018
1 parent 93036d6 commit 321078e
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 27 deletions.
4 changes: 3 additions & 1 deletion presto-orc/src/main/java/com/facebook/presto/orc/DictionaryCompressionOptimizer.java
View file
Expand Up @@ -281,6 +281,8 @@ public interface DictionaryColumn
int getDictionaryBytes();

long convertToDirect();

long getBufferedBytes();
}

private static class DictionaryColumnManager
Expand Down Expand Up @@ -397,7 +399,7 @@ public double getCompressionRatio()

public long getBufferedBytes()
{
return getIndexBytes() + getDictionaryBytes();
return dictionaryColumn.getBufferedBytes();
}
}

Expand Down
52 changes: 26 additions & 26 deletions presto-orc/src/test/java/com/facebook/presto/orc/TestDictionaryCompressionOptimizer.java
View file
Expand Up @@ -55,13 +55,13 @@ public void testNoDictionariesBytesLimit()

// since there are no dictionary columns, the simulator should advance until the strip is full
assertFalse(simulator.isDictionaryMemoryFull());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);

simulator.finalOptimize();

assertFalse(simulator.isDictionaryMemoryFull());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);

simulator.reset();
Expand Down Expand Up @@ -90,14 +90,14 @@ public void testSingleDictionaryColumnRowLimit()
// since there dictionary columns is only 1 MB, the simulator should advance until the strip is full
assertFalse(simulator.isDictionaryMemoryFull());
assertFalse(column.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);

simulator.finalOptimize();

assertFalse(simulator.isDictionaryMemoryFull());
assertFalse(column.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);

simulator.reset();
Expand Down Expand Up @@ -128,14 +128,14 @@ public void testSingleDictionaryColumnByteLimit()
// since there dictionary columns is only 1 MB, the simulator should advance until the strip is full
assertFalse(simulator.isDictionaryMemoryFull());
assertFalse(column.isDirect());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertLessThan(simulator.getRowCount(), expectedMaxRowCount);

simulator.finalOptimize();

assertFalse(simulator.isDictionaryMemoryFull());
assertFalse(column.isDirect());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertLessThan(simulator.getRowCount(), expectedMaxRowCount);

simulator.reset();
Expand Down Expand Up @@ -167,14 +167,14 @@ public void testSingleDictionaryColumnMemoryLimit()
// the simulator should advance until memory is full
assertTrue(simulator.isDictionaryMemoryFull());
assertFalse(column.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);

simulator.finalOptimize();

assertTrue(simulator.isDictionaryMemoryFull());
assertFalse(column.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCount);

simulator.reset();
Expand Down Expand Up @@ -206,22 +206,22 @@ public void testSingleDictionaryColumnMemoryLimitHighlyCompressed()
// the simulator should advance until the dictionary column is converted to direct
assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(column.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip);

simulator.advanceToNextStateChange();

// the simulator should advance until the stripe is full
assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(column.isDirect());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);

simulator.finalOptimize();

assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(column.isDirect());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);

simulator.reset();
Expand Down Expand Up @@ -252,22 +252,22 @@ public void testSingleDirectBytesLimit()
// the simulator should advance until the dictionary column is flipped
assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(column.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip);

simulator.advanceToNextStateChange();

// the simulator should advance until the stripe is full
assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(column.isDirect());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);

simulator.finalOptimize();

assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(column.isDirect());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);

simulator.reset();
Expand Down Expand Up @@ -302,7 +302,7 @@ public void testDictionaryAndDirectBytesLimit()
assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(directColumn.isDirect());
assertFalse(dictionaryColumn.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip);

simulator.advanceToNextStateChange();
Expand All @@ -311,15 +311,15 @@ public void testDictionaryAndDirectBytesLimit()
assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(directColumn.isDirect());
assertFalse(dictionaryColumn.isDirect());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);

simulator.finalOptimize();

assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(directColumn.isDirect());
assertFalse(dictionaryColumn.isDirect());
assertGreaterThanOrEqual(simulator.getBufferedBytes(), stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedMaxRowCountAtFull);

simulator.reset();
Expand Down Expand Up @@ -358,7 +358,7 @@ public void testWideDictionaryAndNarrowDirectBytesLimit()
assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(directColumn.isDirect());
assertFalse(dictionaryColumn.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), expectedRowCountAtFlip);

simulator.advanceToNextStateChange();
Expand All @@ -367,15 +367,15 @@ public void testWideDictionaryAndNarrowDirectBytesLimit()
assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(directColumn.isDirect());
assertFalse(dictionaryColumn.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), maxRowCount);

simulator.finalOptimize();

assertFalse(simulator.isDictionaryMemoryFull());
assertTrue(directColumn.isDirect());
assertFalse(dictionaryColumn.isDirect());
assertLessThan(simulator.getBufferedBytes(), stripeMaxBytes);
assertLessThan(simulator.getBufferedBytes(), (long) stripeMaxBytes);
assertGreaterThanOrEqual(simulator.getRowCount(), maxRowCount);

simulator.reset();
Expand Down Expand Up @@ -423,7 +423,7 @@ public void advanceToNextStateChange()
for (TestDictionaryColumn dictionaryColumn : dictionaryColumns) {
dictionaryColumn.advanceTo(rowCount);
}
optimizer.optimize(getBufferedBytes(), getRowCount());
optimizer.optimize(toIntExact(getBufferedBytes()), getRowCount());
}
}

Expand Down Expand Up @@ -453,11 +453,11 @@ public void reset()
}
}

public int getBufferedBytes()
public long getBufferedBytes()
{
return (rowCount * otherColumnsBytesPerRow) +
return (long) rowCount * otherColumnsBytesPerRow +
dictionaryColumns.stream()
.mapToInt(TestDictionaryColumn::getBufferedBytes)
.mapToLong(TestDictionaryColumn::getBufferedBytes)
.sum();
}

Expand Down Expand Up @@ -526,10 +526,10 @@ public void advanceTo(int rowCount)
this.rowCount = rowCount;
}

public int getBufferedBytes()
public long getBufferedBytes()
{
if (direct) {
return (int) (rowCount * valuesPerRow * bytesPerEntry);
return (long) (rowCount * valuesPerRow * bytesPerEntry);
}
int dictionaryEntries = getDictionaryEntries();
int bytesPerValue = estimateIndexBytesPerValue(dictionaryEntries);
Expand Down

0 comments on commit 321078e

Please sign in to comment.