From 8972039a4c812676fb77ba73b54d5fa3a3d38dc8 Mon Sep 17 00:00:00 2001 From: kumarvishal09 Date: Mon, 30 Jul 2018 21:21:09 +0530 Subject: [PATCH 1/2] Fixed Data loading perfornace issue when number records is high --- .../merger/UnsafeIntermediateMerger.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java index 0b4eae28ee8..62144e0e797 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java @@ -77,8 +77,7 @@ public UnsafeIntermediateMerger(SortParameters parameters) { this.mergedPages = new ArrayList<>(); this.executorService = Executors.newFixedThreadPool(parameters.getNumberOfCores(), new CarbonThreadFactory("UnsafeIntermediatePool:" + parameters.getTableName())); - this.procFiles = - SynchronizedList.decorate(new ArrayList(CarbonCommonConstants.CONSTANT_SIZE_TEN)); + this.procFiles = new ArrayList<>(CarbonCommonConstants.CONSTANT_SIZE_TEN); this.mergerTask = new ArrayList<>(); Integer spillPercentage = CarbonProperties.getInstance().getSortMemorySpillPercentage(); @@ -111,18 +110,15 @@ public void addFileToMerge(File sortTempFile) { } public void startFileMergingIfPossible() { - File[] fileList = null; - synchronized (lockObject) { - if (procFiles.size() >= parameters.getNumberOfIntermediateFileToBeMerged()) { + File[] fileList; + if (procFiles.size() >= parameters.getNumberOfIntermediateFileToBeMerged()) { + synchronized (lockObject) { fileList = procFiles.toArray(new File[procFiles.size()]); this.procFiles = new ArrayList(); - if (LOGGER.isDebugEnabled()) { - LOGGER - .debug("Submitting request for intermediate merging no of files: " + fileList.length); - } } - } - if (null != fileList) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length); + } startIntermediateMerging(fileList); } } From fe9088e74e367c7b0079744a4de20e8154ca6b7e Mon Sep 17 00:00:00 2001 From: kumarvishal09 Date: Mon, 30 Jul 2018 21:23:00 +0530 Subject: [PATCH 2/2] Fixed Data loading perfornace issue when number records is high --- .../loading/sort/unsafe/merger/UnsafeIntermediateMerger.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java index 62144e0e797..18f35d37f65 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java @@ -37,8 +37,6 @@ import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException; import org.apache.carbondata.processing.sort.sortdata.SortParameters; -import org.apache.commons.collections.list.SynchronizedList; - /** * It does mergesort intermediate files to big file. */