From e702650902a8b8ce036f034ed6061c7bb62420c1 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Mon, 27 Jul 2015 15:54:07 +0900 Subject: [PATCH] TAJO-1714: Boosting aggregation performance using GS-Collection --- tajo-core/pom.xml | 13 ++++++++++ .../planner/physical/HashAggregateExec.java | 6 +++-- tajo-project/pom.xml | 25 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index 748950360a..2bf9209c40 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -474,6 +474,19 @@ com.jayway.jsonpath json-path + + com.goldmansachs + gs-collections-api + + + com.goldmansachs + gs-collections + + + com.goldmansachs + gs-collections-testutils + test + diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java index b657622dcd..f713ab870f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java @@ -18,6 +18,8 @@ package org.apache.tajo.engine.planner.physical; +import com.gs.collections.api.map.MutableMap; +import com.gs.collections.impl.map.mutable.UnifiedMap; import org.apache.tajo.engine.planner.KeyProjector; import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.plan.logical.GroupbyNode; @@ -34,7 +36,7 @@ */ public class HashAggregateExec extends AggregationExec { private Tuple tuple = null; - private TupleMap hashTable; + private MutableMap hashTable; private KeyProjector hashKeyProjector; private boolean computed = false; private Iterator> iterator = null; @@ -42,7 +44,7 @@ public class HashAggregateExec extends AggregationExec { public HashAggregateExec(TaskAttemptContext ctx, GroupbyNode plan, PhysicalExec subOp) throws IOException { super(ctx, plan, subOp); hashKeyProjector = new KeyProjector(inSchema, plan.getGroupingColumns()); - hashTable = new TupleMap(10000); + hashTable = UnifiedMap.newMap(); this.tuple = new VTuple(plan.getOutSchema().size()); } diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index a85b61185c..de2729e683 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -40,6 +40,7 @@ 1.1.0 4.0.29.Final 2.6 + 6.2.0 ${project.parent.relativePath}/.. src/main/hadoop-${hadoop.version} @@ -1131,6 +1132,30 @@ json-path 2.0.0 + + com.goldmansachs + gs-collections-api + ${gs.collection} + + + + com.goldmansachs + gs-collections + ${gs.collection} + + + + com.goldmansachs + gs-collections-testutils + ${gs.collection} + test + + + + com.goldmansachs + gs-collections-forkjoin + ${gs.collection} +