From 30a16e3f166ca34c9391a4dbad64bd2a92b58737 Mon Sep 17 00:00:00 2001 From: Keuntae Park Date: Wed, 28 Jan 2015 15:37:07 +0900 Subject: [PATCH] As BaseTupleComparator already handles both ascending and descending sort keys, remove unnecessary DescendingTupleRangeComparator class. - We do not need to consider whether the first sort key is ascending or descending in scheduleRangeShuffledFetches(). - Add TEST_MIN_TASK_NUM support in getNonLeafTaskNum() of Stage.java for Test purpose. --- .../tajo/querymaster/Repartitioner.java | 9 ++--- .../org/apache/tajo/querymaster/Stage.java | 3 +- .../tajo/engine/query/TestSortQuery.java | 34 +++++++++++++++++++ .../TestSortQuery/testSortFirstDesc.sql | 1 + .../TestSortQuery/testSortFirstDesc.result | 10 ++++++ .../org/apache/tajo/storage/TupleRange.java | 9 ----- 6 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql create mode 100644 tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 5a3567453f..7c2d9f4f02 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -725,13 +725,8 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo } } - boolean ascendingFirstKey = sortSpecs[0].isAscending(); SortedMap> map; - if (ascendingFirstKey) { - map = new TreeMap>(); - } else { - map = new TreeMap>(new TupleRange.DescendingTupleRangeComparator()); - } + map = new TreeMap>(); Set fetchSet; try { @@ -740,7 +735,7 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo fetchSet = new HashSet(); for (FetchImpl fetch: fetches) { String rangeParam = - TupleUtil.rangeToQuery(ranges[i], ascendingFirstKey ? i == (ranges.length - 1) : i == 0, encoder); + TupleUtil.rangeToQuery(ranges[i], i == (ranges.length - 1) , encoder); FetchImpl copy = null; try { copy = fetch.clone(); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 208d4a6240..916d8c790b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -988,7 +988,8 @@ public static int getNonLeafTaskNum(Stage stage) { int mb = (int) Math.ceil((double)volume / 1048576); LOG.info(stage.getId() + ", Table's volume is approximately " + mb + " MB"); // determine the number of task per 64MB - int maxTaskNum = Math.max(1, (int) Math.ceil((double)mb / 64)); + int minTaskNum = Math.max(1, stage.getContext().getQueryMasterContext().getConf().getInt(ConfVars.$TEST_MIN_TASK_NUM.varname, 1)); + int maxTaskNum = Math.max(minTaskNum, (int) Math.ceil((double)mb / 64)); LOG.info(stage.getId() + ", The determined number of non-leaf tasks is " + maxTaskNum); return maxTaskNum; } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java index 9c911f9930..ff911778ff 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java @@ -102,6 +102,40 @@ public final void testSortDesc() throws Exception { cleanupQuery(res); } + + @Test + public final void testSortFirstDesc() throws Exception { + try { + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2"); + KeyValueSet tableOptions = new KeyValueSet(); + tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + tableOptions.set(StorageConstants.TEXT_NULL, "\\\\N"); + + Schema schema = new Schema(); + schema.addColumn("col1", Type.INT4); + schema.addColumn("col2", Type.TEXT); + String[] data = new String[]{ + "1|abc", + "3|dfa", + "3|das", + "1|abb", + "1|abc", + "3|dfb", + "3|dat", + "1|abe" + }; + TajoTestingCluster.createTable("sortfirstdesc", schema, tableOptions, data, 2); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "0"); + executeString("DROP TABLE sortfirstdesc PURGE;").close(); + } + } + + @Test public final void testTopK() throws Exception { ResultSet res = executeQuery(); diff --git a/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql b/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql new file mode 100644 index 0000000000..644feb0e14 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSortQuery/testSortFirstDesc.sql @@ -0,0 +1 @@ +select col1, col2 from sortfirstdesc order by col1 desc, col2; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result b/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result new file mode 100644 index 0000000000..2ed9235b27 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSortQuery/testSortFirstDesc.result @@ -0,0 +1,10 @@ +col1,col2 +------------------------------- +3,das +3,dat +3,dfa +3,dfb +1,abb +1,abc +1,abc +1,abe \ No newline at end of file diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java index e824b99ae7..043409a50c 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TupleRange.java @@ -94,15 +94,6 @@ public int compareTo(TupleRange o) { } } - public static class DescendingTupleRangeComparator - implements Comparator { - - @Override - public int compare(TupleRange left, TupleRange right) { - return right.compareTo(left); - } - } - public TupleRange clone() throws CloneNotSupportedException { TupleRange newRange = (TupleRange) super.clone(); newRange.setStart(start.clone());