From 54631ec6eb2a78445ee53f154d82b1206253856c Mon Sep 17 00:00:00 2001 From: xuchuanyin Date: Wed, 14 Mar 2018 14:45:53 +0800 Subject: [PATCH] Reduce massive object generation in global sort generate compatator outside the function, otherwise it will be generated for every row --- .../spark/load/DataLoadProcessBuilderOnSpark.scala | 13 ++++++------- .../processing/sort/sortdata/NewRowComparator.java | 4 +++- .../sortdata/NewRowComparatorForNormalDims.java | 5 ++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala index 1062cd72c77..dc238fb383c 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala @@ -80,15 +80,14 @@ object DataLoadProcessBuilderOnSpark { // 3. Sort val configuration = DataLoadProcessBuilder.createConfiguration(model) val sortParameters = SortParameters.createSortParameters(configuration) + val rowComparator: Comparator[Array[AnyRef]] = + if (sortParameters.getNoDictionaryCount > 0) { + new NewRowComparator(sortParameters.getNoDictionaryDimnesionColumn) + } else { + new NewRowComparatorForNormalDims(sortParameters.getDimColCount) + } object RowOrdering extends Ordering[Array[AnyRef]] { def compare(rowA: Array[AnyRef], rowB: Array[AnyRef]): Int = { - val rowComparator: Comparator[Array[AnyRef]] = - if (sortParameters.getNoDictionaryCount > 0) { - new NewRowComparator(sortParameters.getNoDictionaryDimnesionColumn) - } else { - new NewRowComparatorForNormalDims(sortParameters.getDimColCount) - } - rowComparator.compare(rowA, rowB) } } diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java index 3f945333e88..f47ecc70258 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java @@ -17,11 +17,13 @@ package org.apache.carbondata.processing.sort.sortdata; +import java.io.Serializable; import java.util.Comparator; import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer; -public class NewRowComparator implements Comparator { +public class NewRowComparator implements Comparator, Serializable { + private static final long serialVersionUID = -1739874611112709436L; /** * mapping of dictionary dimensions and no dictionary of sort_column. diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java index 7538c9233f2..aea83baf264 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java @@ -16,13 +16,16 @@ */ package org.apache.carbondata.processing.sort.sortdata; +import java.io.Serializable; import java.util.Comparator; /** * This class is used as comparator for comparing dims which are non high cardinality dims. * Here the dims will be in form of int[] (surrogates) so directly comparing the integers. */ -public class NewRowComparatorForNormalDims implements Comparator { +public class NewRowComparatorForNormalDims implements Comparator, Serializable { + private static final long serialVersionUID = -1749874611112709432L; + /** * dimension count */