Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Kylin 3597 fix sonar issues #398

Merged
merged 1 commit into from
Dec 26, 2018
Merged

Kylin 3597 fix sonar issues #398

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 32 additions & 31 deletions core-cube/src/main/java/org/apache/kylin/cube/inmemcubing2/InMemCubeBuilder2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ private <T> NavigableMap<Long, CuboidResult> buildAndCollect(final RecordConsume
final ICuboidResultListener listener) throws IOException {

long startTime = System.currentTimeMillis();
logger.info("In Mem Cube Build2 start, " + cubeDesc.getName());
logger.info("In Mem Cube Build2 start, {}", cubeDesc.getName());

// build base cuboid
buildBaseCuboid(input, listener);
Expand All @@ -172,8 +172,8 @@ public void run() {
rootTask.join();

long endTime = System.currentTimeMillis();
logger.info("In Mem Cube Build2 end, " + cubeDesc.getName() + ", takes " + (endTime - startTime) + " ms");
logger.info("total CuboidResult count:" + resultCollector.getAllResult().size());
logger.info("In Mem Cube Build2 end, {}, takes {} ms", cubeDesc.getName(), (endTime - startTime));
logger.info("total CuboidResult count: {}", resultCollector.getAllResult().size());
return resultCollector.getAllResult();
}

Expand Down Expand Up @@ -263,26 +263,26 @@ public Queue<CuboidTask> getCompletedTaskQueue() {

private void makeMemoryBudget() {
int systemAvailMB = MemoryBudgetController.gcAndGetSystemAvailMB();
logger.info("System avail " + systemAvailMB + " MB");
logger.info("System avail {} MB", systemAvailMB);
int reserve = reserveMemoryMB;
logger.info("Reserve " + reserve + " MB for system basics");
logger.info("Reserve {} MB for system basics", reserve);

int budget = systemAvailMB - reserve;
if (budget < baseResult.aggrCacheMB) {
// make sure we have base aggr cache as minimal
budget = baseResult.aggrCacheMB;
logger.warn("System avail memory (" + systemAvailMB + " MB) is less than base aggr cache ("
+ baseResult.aggrCacheMB + " MB) + minimal reservation (" + reserve
+ " MB), consider increase JVM heap -Xmx");
logger.warn(
"System avail memory ({} MB) is less than base aggr cache ({} MB) + minimal reservation ({} MB), consider increase JVM heap -Xmx",
systemAvailMB, baseResult.aggrCacheMB, reserve);
}

logger.info("Memory Budget is " + budget + " MB");
logger.info("Memory Budget is {} MB", budget);
memBudget = new MemoryBudgetController(budget);
}

private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T> input,
MemoryBudgetController.MemoryWaterLevel baseCuboidMemTracker) throws IOException {
logger.info("Calculating base cuboid " + baseCuboidId);
logger.info("Calculating base cuboid {}", baseCuboidId);

Stopwatch sw = new Stopwatch();
sw.start();
Expand All @@ -295,29 +295,30 @@ private <T> CuboidResult createBaseCuboid(RecordConsumeBlockingQueueController<T
GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null)
.setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond())
.setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest();
GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req);
aggregationScanner.trackMemoryLevel(baseCuboidMemTracker);
try (GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req)) {
aggregationScanner.trackMemoryLevel(baseCuboidMemTracker);

int count = 0;
for (GTRecord r : aggregationScanner) {
if (count == 0) {
baseCuboidMemTracker.markHigh();
int count = 0;
for (GTRecord r : aggregationScanner) {
if (count == 0) {
baseCuboidMemTracker.markHigh();
}
baseBuilder.write(r);
count++;
}
baseBuilder.write(r);
count++;
}
aggregationScanner.close();
baseBuilder.close();

sw.stop();
logger.info("Cuboid " + baseCuboidId + " has " + count + " rows, build takes " + sw.elapsedMillis() + "ms");
baseBuilder.close();

sw.stop();
logger.info("Cuboid {} has {} rows, build takes {}ms", baseCuboidId, count, sw.elapsedMillis());

int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache()
/ MemoryBudgetController.ONE_MB);
logger.info("Wild estimate of base aggr cache is " + mbEstimateBaseAggrCache + " MB");
int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache()
/ MemoryBudgetController.ONE_MB);
logger.info("Wild estimate of base aggr cache is {} MB", mbEstimateBaseAggrCache);

return updateCuboidResult(baseCuboidId, baseCuboid, count, sw.elapsedMillis(), 0,
input.inputConverterUnit.ifChange());
return updateCuboidResult(baseCuboidId, baseCuboid, count, sw.elapsedMillis(), 0,
input.inputConverterUnit.ifChange());
}
}

private CuboidResult updateCuboidResult(long cuboidId, GridTable table, int nRows, long timeSpent,
Expand Down Expand Up @@ -364,7 +365,7 @@ private GTAggregateScanner prepareGTAggregationScanner(GridTable gridTable, long
aggrMask[i] = !measureDescs[i].getFunction().getMeasureType().onlyAggrInBaseCuboid();

if (!aggrMask[i]) {
logger.info(measureDescs[i].toString() + " doesn't need aggregation.");
logger.info("{} doesn't need aggregation.", measureDescs[i]);
}
}
scanner.setAggrMask(aggrMask);
Expand All @@ -377,7 +378,7 @@ protected CuboidResult scanAndAggregateGridTable(GridTable gridTable, GridTable
long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException {
Stopwatch sw = new Stopwatch();
sw.start();
logger.info("Calculating cuboid " + cuboidId);
logger.info("Calculating cuboid {}", cuboidId);

GTAggregateScanner scanner = prepareGTAggregationScanner(gridTable, parentId, cuboidId, aggregationColumns,
measureColumns);
Expand All @@ -401,7 +402,7 @@ protected CuboidResult scanAndAggregateGridTable(GridTable gridTable, GridTable
builder.close();
}
sw.stop();
logger.info("Cuboid " + cuboidId + " has " + count + " rows, build takes " + sw.elapsedMillis() + "ms");
logger.info("Cuboid {} has {} rows, build takes {}ms", cuboidId, count, sw.elapsedMillis());

return updateCuboidResult(cuboidId, newGridTable, count, sw.elapsedMillis(), 0);
}
Expand Down