Skip to content

Commit

Permalink
Fix test
Browse files Browse the repository at this point in the history
  • Loading branch information
aokolnychyi committed Jun 7, 2023
1 parent 419245e commit b537560
Showing 1 changed file with 7 additions and 10 deletions.
17 changes: 7 additions & 10 deletions core/src/test/java/org/apache/iceberg/util/TestTableScanUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -84,26 +84,23 @@ public void testAdaptiveSplitSize() {
long minSplitSize = 16 * 1024 * 1024; // 16 MB
long maxSplitSize = 512 * 1024 * 1024; // 512 MB

// targeting a split per slot with 80 slots would mean having splits with 9+ GB of data
// targeting 2 splits per slot with 80 slots would mean having splits with 4.5+ GB of data
// that's why the utility would target for 19 * 80 = 1520 splits
// it is the smallest number of splits that would produce a split size smaller than the maximum
int smallParallelism = 80;
long expectedSplitSizeSmallParallelism = (long) (1.1 * (scanSize / 1520));
Assert.assertEquals(
"Split size must match",
expectedSplitSizeSmallParallelism,
(long) (1.1 * (scanSize / 1520)),
TableScanUtil.computeSplitSize(scanSize, smallParallelism, minSplitSize, maxSplitSize));

// targeting a split per slot with 5K slots would produce well-sized splits
int reasonableParallelism = 5_000;
long expectedSplitSizeReasonableParallelism = (long) (1.1 * (scanSize / reasonableParallelism));
// targeting 2 splits per slot with 5K slots would produce well-sized splits
int mediumParallelism = 5_000;
Assert.assertEquals(
"Split size must match",
expectedSplitSizeReasonableParallelism,
TableScanUtil.computeSplitSize(
scanSize, reasonableParallelism, minSplitSize, maxSplitSize));
(long) (1.1 * (scanSize / (2 * mediumParallelism))),
TableScanUtil.computeSplitSize(scanSize, mediumParallelism, minSplitSize, maxSplitSize));

// targeting a split per slot with 100K slots would produce tiny splits
// targeting 2 splits per slot with 100K slots would produce tiny splits
// that's why the min split size is used
int bigParallelism = 100_000;
Assert.assertEquals(
Expand Down

0 comments on commit b537560

Please sign in to comment.