Merge c14a937 into c65cc12

apache · Jul 25, 2019 · 4236726 · 4236726
2 parents c65cc12 + c14a937
commit 4236726
Showing 1 changed file with 10 additions and 3 deletions.
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -207,9 +207,6 @@ private List<ExtendedBlocklet> pruneMultiThread(List<Segment> segments,
      */
 
     int numOfThreadsForPruning = CarbonProperties.getNumOfThreadsForPruning();
-    LOG.info(
-        "Number of threads selected for multi-thread block pruning is " + numOfThreadsForPruning
-            + ". total files: " + totalFiles + ". total segments: " + segments.size());
     int filesPerEachThread = totalFiles / numOfThreadsForPruning;
     int prev;
     int filesCount = 0;
@@ -247,6 +244,16 @@ private List<ExtendedBlocklet> pruneMultiThread(List<Segment> segments,
             .add(new SegmentDataMapGroup(segment, prev, eachSegmentDataMapList.size() - 1));
       }
     }
+    if (datamapListForEachThread.size() < numOfThreadsForPruning) {
+      // If the total datamaps less than 2 * numOfThreadsForPruning,
+      // chances that all datamaps is distributed before last thread itself.
+      // so, launch only that many threads.
+      LOG.info("Datamaps is distributed in " + datamapListForEachThread.size() + " threads");
+      numOfThreadsForPruning = datamapListForEachThread.size();
+    }
+    LOG.info(
+        "Number of threads selected for multi-thread block pruning is " + numOfThreadsForPruning
+            + ". total files: " + totalFiles + ". total segments: " + segments.size());
     // adding the last segmentList data
     datamapListForEachThread.add(segmentDataMapGroupList);
     processedFileCount += filesCount;