From a3b78642abb6c160b147eae2f29a10e362c14cac Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 8 Jul 2014 17:47:42 +0900 Subject: [PATCH 01/13] Improve session variables to affect the query config. --- .../engine/planner/PhysicalPlannerImpl.java | 12 ++--- .../engine/planner/global/GlobalPlanner.java | 10 ++-- .../tajo/engine/query/QueryContext.java | 46 +++++++++++++++++-- .../master/querymaster/Repartitioner.java | 6 ++- .../tajo/master/querymaster/SubQuery.java | 15 +++--- .../tajo/worker/TaskAttemptContext.java | 4 ++ 6 files changed, 70 insertions(+), 23 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index f41d61d679..04a785609c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -37,6 +37,7 @@ import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.physical.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InternalException; import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer; @@ -68,7 +69,6 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { private static final Log LOG = LogFactory.getLog(PhysicalPlannerImpl.class); private static final int UNGENERATED_PID = -1; - private final long INNER_JOIN_INMEMORY_HASH_THRESHOLD; protected final TajoConf conf; protected final AbstractStorageManager sm; @@ -76,8 +76,6 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { public PhysicalPlannerImpl(final TajoConf conf, final AbstractStorageManager sm) { this.conf = conf; this.sm = sm; - - this.INNER_JOIN_INMEMORY_HASH_THRESHOLD = conf.getLongVar(ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD); } public PhysicalExec createPlan(final TaskAttemptContext context, final LogicalNode logicalPlan) @@ -252,7 +250,7 @@ public boolean checkIfInMemoryInnerJoinIsPossible(TaskAttemptContext context, Lo throws IOException { String [] lineage = PlannerUtil.getRelationLineage(node); long volume = estimateSizeRecursive(context, lineage); - boolean inMemoryInnerJoinFlag = volume <= INNER_JOIN_INMEMORY_HASH_THRESHOLD; + boolean inMemoryInnerJoinFlag = volume <= QueryContext.getLongVar(context.getQueryContext(), conf, ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD); LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.", context.getTaskId().toString(), (left ? "Left" : "Right"), @@ -465,7 +463,7 @@ private PhysicalExec createBestLeftOuterJoinPlan(TaskAttemptContext context, Joi String [] rightLineage = PlannerUtil.getRelationLineage(plan.getRightChild()); long rightTableVolume = estimateSizeRecursive(context, rightLineage); - if (rightTableVolume < conf.getLongVar(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)) { + if (rightTableVolume < QueryContext.getLongVar(context.getQueryContext(), conf, ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)) { // we can implement left outer join using hash join, using the right operand as the build relation LOG.info("Left Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, leftExec, rightExec); @@ -483,7 +481,7 @@ private PhysicalExec createBestRightJoinPlan(TaskAttemptContext context, JoinNod // blocking, but merge join is blocking as well) String [] outerLineage4 = PlannerUtil.getRelationLineage(plan.getLeftChild()); long outerSize = estimateSizeRecursive(context, outerLineage4); - if (outerSize < conf.getLongVar(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)){ + if (outerSize < QueryContext.getLongVar(context.getQueryContext(), conf, ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)){ LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, rightExec, leftExec); } else { @@ -961,7 +959,7 @@ private PhysicalExec createBestAggregationPlan(TaskAttemptContext context, Group String [] outerLineage = PlannerUtil.getRelationLineage(groupbyNode.getChild()); long estimatedSize = estimateSizeRecursive(context, outerLineage); - final long threshold = conf.getLongVar(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD); + final long threshold = QueryContext.getLongVar(context.getQueryContext(), conf, ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD); // if the relation size is less than the threshold, // the hash aggregation will be used. diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index edd5674f84..3360a41ac3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -38,6 +38,7 @@ import org.apache.tajo.engine.planner.global.builder.DistinctGroupbyBuilder; import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InternalException; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; @@ -120,7 +121,7 @@ public void build(MasterPlan masterPlan) throws IOException, PlanningException { LogicalNode inputPlan = PlannerUtil.clone(masterPlan.getLogicalPlan(), masterPlan.getLogicalPlan().getRootBlock().getRoot()); - boolean autoBroadcast = conf.getBoolVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); + boolean autoBroadcast = QueryContext.getBoolVar(masterPlan.getContext(), conf, TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); if (autoBroadcast) { // pre-visit finding broadcast join target table // this visiting doesn't make any execution block and change plan @@ -263,11 +264,12 @@ private ExecutionBlock buildJoinPlan(GlobalPlanContext context, JoinNode joinNod MasterPlan masterPlan = context.plan; ExecutionBlock currentBlock; - boolean autoBroadcast = conf.getBoolVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); + boolean autoBroadcast = + QueryContext.getBoolVar(context.plan.getContext(), conf, TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); // to check when the tajo.dist-query.join.broadcast.auto property is true if (autoBroadcast && joinNode.isCandidateBroadcast()) { - long broadcastThreshold = conf.getLongVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD); + long broadcastThreshold = QueryContext.getLongVar(context.plan.getContext(), conf, TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD); List broadtargetTables = new ArrayList(); int numLargeTables = 0; for(LogicalNode eachNode: joinNode.getBroadcastTargets()) { @@ -310,7 +312,7 @@ private ExecutionBlock buildJoinPlan(GlobalPlanContext context, JoinNode joinNod TableDesc leftDesc = leftScan.getTableDesc(); TableDesc rightDesc = rightScan.getTableDesc(); - long broadcastThreshold = conf.getLongVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD); + long broadcastThreshold = QueryContext.getLongVar(context.getPlan().getContext(), conf, TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD); if (getTableVolume(leftScan) < broadcastThreshold) { leftBroadcasted = true; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 79d6cb3032..879b2df47e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -18,6 +18,8 @@ package org.apache.tajo.engine.query; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -27,6 +29,8 @@ import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; public class QueryContext extends KeyValueSet { + private static final Log LOG = LogFactory.getLog(QueryContext.class); + public static final String COMMAND_TYPE = "tajo.query.command"; public static final String STAGING_DIR = "tajo.query.staging_dir"; @@ -39,8 +43,8 @@ public class QueryContext extends KeyValueSet { public static final String OUTPUT_OVERWRITE = "tajo.query.output.overwrite"; public static final String OUTPUT_AS_DIRECTORY = "tajo.query.output.asdirectory"; - public static final String TRUE_VALUE = "1"; - public static final String FALSE_VALUE = "0"; + public static final String TRUE_VALUE = "true"; + public static final String FALSE_VALUE = "false"; public QueryContext() {} @@ -66,7 +70,7 @@ public void setBool(String key, boolean val) { public boolean getBool(String key) { String strVal = get(key); - return strVal != null ? strVal.equals(TRUE_VALUE) : false; + return strVal != null ? strVal.equalsIgnoreCase(TRUE_VALUE) : false; } public void setUser(String username) { @@ -188,4 +192,40 @@ public void setHiveQueryMode() { public boolean isHiveQueryMode() { return getBool("hive.query.mode"); } + + public static boolean getBoolVar(QueryContext context, TajoConf conf, TajoConf.ConfVars key) { + if (context.get(key.varname) != null) { + return context.getBool(key.varname); + } else { + return conf.getBoolVar(key); + } + } + + public static Integer getIntVar(QueryContext context, TajoConf conf, TajoConf.ConfVars key) { + if (context.get(key.varname) != null) { + String val = context.get(key.varname); + try { + return Integer.valueOf(val); + } catch (NumberFormatException nfe) { + LOG.warn(nfe.getMessage()); + return conf.getIntVar(key); + } + } else { + return conf.getIntVar(key); + } + } + + public static Long getLongVar(QueryContext context, TajoConf conf, TajoConf.ConfVars key) { + if (context.get(key.varname) != null) { + String val = context.get(key.varname); + try { + return Long.valueOf(val); + } catch (NumberFormatException nfe) { + LOG.warn(nfe.getMessage()); + return conf.getLongVar(key); + } + } else { + return conf.getLongVar(key); + } + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index 0046dbe13f..e4abd7f794 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -37,6 +37,7 @@ import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.utils.TupleUtil; import org.apache.tajo.exception.InternalException; import org.apache.tajo.ipc.TajoWorkerProtocol; @@ -271,8 +272,9 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC // Getting the desire number of join tasks according to the volumn // of a larger table int largerIdx = stats[0] >= stats[1] ? 0 : 1; - int desireJoinTaskVolumn = subQuery.getContext().getConf(). - getIntVar(ConfVars.DIST_QUERY_JOIN_TASK_VOLUME); + int desireJoinTaskVolumn = + QueryContext.getIntVar(subQuery.getContext().getQueryContext(), subQuery.getContext().getConf(), + ConfVars.DIST_QUERY_JOIN_TASK_VOLUME); // calculate the number of tasks according to the data size int mb = (int) Math.ceil((double)stats[largerIdx] / 1048576); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index d4c94e815f..9b9766a54c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -52,6 +52,7 @@ import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.engine.planner.logical.ScanNode; import org.apache.tajo.engine.planner.logical.StoreTableNode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.TajoMasterProtocol; import org.apache.tajo.master.*; import org.apache.tajo.master.TaskRunnerGroupEvent.EventType; @@ -741,16 +742,16 @@ public static int calculateShuffleOutputNum(SubQuery subQuery, DataChannel chann LOG.info(subQuery.getId() + ", Bigger Table's volume is approximately " + mb + " MB"); int taskNum = (int) Math.ceil((double) mb / - conf.getIntVar(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME)); + QueryContext.getIntVar(subQuery.getContext().getQueryContext(), conf, ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME)); int totalMem = getClusterTotalMemory(subQuery); LOG.info(subQuery.getId() + ", Total memory of cluster is " + totalMem + " MB"); - int slots = Math.max(totalMem / conf.getIntVar(ConfVars.TASK_DEFAULT_MEMORY), 1); + int slots = Math.max(totalMem / QueryContext.getIntVar(subQuery.getContext().getQueryContext(), conf, ConfVars.TASK_DEFAULT_MEMORY), 1); // determine the number of task taskNum = Math.min(taskNum, slots); - if (conf.getIntVar(ConfVars.TESTCASE_MIN_TASK_NUM) > 0) { - taskNum = conf.getIntVar(ConfVars.TESTCASE_MIN_TASK_NUM); + if ( QueryContext.getIntVar(subQuery.getContext().getQueryContext(), conf, ConfVars.TESTCASE_MIN_TASK_NUM) > 0) { + taskNum = QueryContext.getIntVar(subQuery.getContext().getQueryContext(), conf, ConfVars.TESTCASE_MIN_TASK_NUM); LOG.warn("!!!!! TESTCASE MODE !!!!!"); } LOG.info(subQuery.getId() + ", The determined number of join partitions is " + taskNum); @@ -787,12 +788,12 @@ public static int calculateShuffleOutputNum(SubQuery subQuery, DataChannel chann LOG.info(subQuery.getId() + ", Table's volume is approximately " + mb + " MB"); // determine the number of task int taskNumBySize = (int) Math.ceil((double) mb / - conf.getIntVar(ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME)); + QueryContext.getIntVar(subQuery.getContext().getQueryContext(), conf, ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME)); int totalMem = getClusterTotalMemory(subQuery); LOG.info(subQuery.getId() + ", Total memory of cluster is " + totalMem + " MB"); - int slots = Math.max(totalMem / conf.getIntVar(ConfVars.TASK_DEFAULT_MEMORY), 1); + int slots = Math.max(totalMem / QueryContext.getIntVar(subQuery.getContext().getQueryContext(), conf, ConfVars.TASK_DEFAULT_MEMORY), 1); int taskNum = Math.min(taskNumBySize, slots); //Maximum partitions LOG.info(subQuery.getId() + ", The determined number of aggregation partitions is " + taskNum); return taskNum; @@ -926,7 +927,7 @@ private static void scheduleFragmentsForLeafQuery(SubQuery subQuery) throws IOEx subQuery.schedulerContext.setEstimatedTaskNum(fragments.size()); } else { TajoConf conf = subQuery.context.getConf(); - subQuery.schedulerContext.setTaskSize(conf.getIntVar(ConfVars.TASK_DEFAULT_SIZE) * 1024 * 1024); + subQuery.schedulerContext.setTaskSize( QueryContext.getIntVar(subQuery.getContext().getQueryContext(), conf, ConfVars.TASK_DEFAULT_SIZE) * 1024 * 1024); int estimatedTaskNum = (int) Math.ceil((double) table.getStats().getNumBytes() / (double) subQuery.schedulerContext.getTaskSize()); subQuery.schedulerContext.setEstimatedTaskNum(estimatedTaskNum); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index b1246ece1f..945fb9672b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -105,6 +105,10 @@ public TaskAttemptContext(TajoConf conf, QueryContext queryContext, final QueryU public TajoConf getConf() { return this.conf; } + + public String getConfig(String key) { + return queryContext.get(key) != null ? queryContext.get(key) : conf.get(key); + } public TaskAttemptState getState() { return this.state; From 0a0035d9b259a1a05ba790b7a778a745251d27bd Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 8 Jul 2014 21:54:32 +0900 Subject: [PATCH 02/13] Fixed. --- .../planner/physical/StoreTableExec.java | 65 ++++++++++++++++--- .../tajo/engine/query/QueryContext.java | 1 + 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index b1d0400abe..a8be79172d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -18,15 +18,19 @@ package org.apache.tajo.engine.planner.physical; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.statistics.StatisticsUtil; +import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.logical.InsertNode; import org.apache.tajo.engine.planner.logical.PersistentStoreNode; -import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.StorageManagerFactory; -import org.apache.tajo.storage.Tuple; +import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.storage.*; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; @@ -35,40 +39,66 @@ * This is a physical executor to store a table part into a specified storage. */ public class StoreTableExec extends UnaryPhysicalExec { + private static Log LOG = LogFactory.getLog(StoreTableExec.class); + private PersistentStoreNode plan; + private TableMeta meta; private Appender appender; private Tuple tuple; + private TableStats sumStats; + private long maxPerFileSize = Long.MAX_VALUE; + private int writtenFileNum = 0; + private FileSystem fs; + private Path lastFileName; public StoreTableExec(TaskAttemptContext context, PersistentStoreNode plan, PhysicalExec child) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema(), child); this.plan = plan; + + if (context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE) != null) { + maxPerFileSize = Long.valueOf(context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE)); + LOG.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>> PER FILE SIZE:" + maxPerFileSize); + } else { + LOG.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>> EMPTY PER FILE SIZE:"); + } } public void init() throws IOException { super.init(); - TableMeta meta; if (plan.hasOptions()) { meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions()); } else { meta = CatalogUtil.newTableMeta(plan.getStorageType()); } + fs = context.getOutputPath().getFileSystem(context.getConf()); + openNewFile(writtenFileNum); + } + + public void openNewFile(int suffixId) throws IOException { + lastFileName = context.getOutputPath(); + if (suffixId > 0) { + lastFileName = new Path(lastFileName, "_" + suffixId); + } + if (plan instanceof InsertNode) { InsertNode createTableNode = (InsertNode) plan; appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, - createTableNode.getTableSchema(), context.getOutputPath()); + createTableNode.getTableSchema(), lastFileName); } else { String nullChar = context.getQueryContext().get(ConfVars.CSVFILE_NULL.varname, ConfVars.CSVFILE_NULL.defaultVal); meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, - context.getOutputPath()); + lastFileName); } appender.enableStats(); appender.init(); } + long writtenSize = 0; + /* (non-Javadoc) * @see PhysicalExec#next() */ @@ -76,6 +106,21 @@ public void init() throws IOException { public Tuple next() throws IOException { while((tuple = child.next()) != null) { appender.addTuple(tuple); + + if (writtenSize > maxPerFileSize) { + appender.close(); + writtenFileNum++; + + if (sumStats == null) { + sumStats = appender.getStats(); + } else { + StatisticsUtil.aggregateTableStat(sumStats, appender.getStats()); + } + openNewFile(writtenFileNum); + LOG.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>> File Flushed" + maxPerFileSize); + writtenSize = 0; + } + writtenSize += MemoryUtil.calculateMemorySize(tuple); } return null; @@ -93,7 +138,11 @@ public void close() throws IOException { appender.flush(); appender.close(); // Collect statistics data - context.setResultStats(appender.getStats()); + if (sumStats == null) { + context.setResultStats(appender.getStats()); + } else { + context.setResultStats(sumStats); + } if (context.getTaskId() != null) { context.addShuffleFileOutput(0, context.getTaskId().toString()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 879b2df47e..7a28d3843c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -42,6 +42,7 @@ public class QueryContext extends KeyValueSet { public static final String OUTPUT_PARTITIONS = "tajo.query.output.partitions"; public static final String OUTPUT_OVERWRITE = "tajo.query.output.overwrite"; public static final String OUTPUT_AS_DIRECTORY = "tajo.query.output.asdirectory"; + public static final String OUTPUT_PER_FILE_SIZE = "tajo.query.output.perfile-size-mb"; public static final String TRUE_VALUE = "true"; public static final String FALSE_VALUE = "false"; From 3fb54a6dde89d2d8e972253c1eccd17f334180d4 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 9 Jul 2014 11:23:28 +0900 Subject: [PATCH 03/13] Completed output file rotating. --- .../SortBasedColPartitionStoreExec.java | 85 ++++++++++++++----- .../planner/physical/StoreTableExec.java | 20 ++--- .../tajo/engine/query/QueryContext.java | 2 +- 3 files changed, 74 insertions(+), 33 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java index 8c55d7f503..7665e7a00c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java @@ -28,12 +28,12 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; +import org.apache.tajo.engine.planner.logical.InsertNode; import org.apache.tajo.engine.planner.logical.StoreTableNode; -import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.StorageManagerFactory; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; +import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.storage.*; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; @@ -51,9 +51,19 @@ public class SortBasedColPartitionStoreExec extends ColPartitionStoreExec { private Appender appender; private TableStats aggregated; + // for file rotating + private long maxPerFileSize = Long.MAX_VALUE; + private int writtenFileNum = 0; + private Path lastFileName; + private long writtenTupleSize = 0; + public SortBasedColPartitionStoreExec(TaskAttemptContext context, StoreTableNode plan, PhysicalExec child) throws IOException { super(context, plan, child); + + if (context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE) != null) { + maxPerFileSize = Long.valueOf(context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE)); + } } public void init() throws IOException { @@ -63,30 +73,50 @@ public void init() throws IOException { aggregated = new TableStats(); } - private Appender getAppender(String partition) throws IOException { - Path dataFile = getDataFile(partition); - FileSystem fs = dataFile.getFileSystem(context.getConf()); + private Appender getAppenderForNewPartition(String partition) throws IOException { + lastFileName = getDataFile(partition); + FileSystem fs = lastFileName.getFileSystem(context.getConf()); - if (fs.exists(dataFile.getParent())) { - LOG.info("Path " + dataFile.getParent() + " already exists!"); + if (fs.exists(lastFileName.getParent())) { + LOG.info("Path " + lastFileName.getParent() + " already exists!"); } else { - fs.mkdirs(dataFile.getParent()); - LOG.info("Add subpartition path directory :" + dataFile.getParent()); + fs.mkdirs(lastFileName.getParent()); + LOG.info("Add subpartition path directory :" + lastFileName.getParent()); } - if (fs.exists(dataFile)) { - LOG.info("File " + dataFile + " already exists!"); - FileStatus status = fs.getFileStatus(dataFile); + if (fs.exists(lastFileName)) { + LOG.info("File " + lastFileName + " already exists!"); + FileStatus status = fs.getFileStatus(lastFileName); LOG.info("File size: " + status.getLen()); } - appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, dataFile); - appender.enableStats(); - appender.init(); + openNewFile(0); return appender; } + public void openNewFile(int suffixId) throws IOException { + Path actualFilePath = lastFileName; + if (suffixId > 0) { + actualFilePath = new Path(lastFileName + "_" + suffixId); + } + + if (plan instanceof InsertNode) { + InsertNode createTableNode = (InsertNode) plan; + appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, + createTableNode.getTableSchema(), actualFilePath); + } else { + String nullChar = context.getQueryContext().get(TajoConf.ConfVars.CSVFILE_NULL.varname, + TajoConf.ConfVars.CSVFILE_NULL.defaultVal); + meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); + appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, + actualFilePath); + } + + appender.enableStats(); + appender.init(); + } + private void fillKeyTuple(Tuple inTuple, Tuple keyTuple) { for (int i = 0; i < keyIds.length; i++) { keyTuple.put(i, inTuple.get(keyIds[i])); @@ -115,19 +145,34 @@ public Tuple next() throws IOException { fillKeyTuple(tuple, currentKey); if (prevKey == null) { - appender = getAppender(getSubdirectory(currentKey)); + appender = getAppenderForNewPartition(getSubdirectory(currentKey)); prevKey = new VTuple(currentKey); } else { - if (!prevKey.equals(currentKey)) { + + if (!prevKey.equals(currentKey)) { // new partition appender.close(); StatisticsUtil.aggregateTableStat(aggregated, appender.getStats()); - appender = getAppender(getSubdirectory(currentKey)); + appender = getAppenderForNewPartition(getSubdirectory(currentKey)); prevKey = new VTuple(currentKey); + + // reset all states for file rotating + writtenTupleSize = 0; + writtenFileNum = 0; } } appender.addTuple(tuple); + writtenTupleSize += MemoryUtil.calculateMemorySize(tuple); + + if (writtenTupleSize > maxPerFileSize) { + appender.close(); + writtenFileNum++; + StatisticsUtil.aggregateTableStat(aggregated, appender.getStats()); + + openNewFile(writtenFileNum); + writtenTupleSize = 0; + } } return null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index a8be79172d..294cb9f44e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -46,10 +46,12 @@ public class StoreTableExec extends UnaryPhysicalExec { private Appender appender; private Tuple tuple; private TableStats sumStats; + + // for file rotating private long maxPerFileSize = Long.MAX_VALUE; private int writtenFileNum = 0; - private FileSystem fs; private Path lastFileName; + private long writtenTupleSize = 0; public StoreTableExec(TaskAttemptContext context, PersistentStoreNode plan, PhysicalExec child) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema(), child); @@ -57,9 +59,6 @@ public StoreTableExec(TaskAttemptContext context, PersistentStoreNode plan, Phys if (context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE) != null) { maxPerFileSize = Long.valueOf(context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE)); - LOG.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>> PER FILE SIZE:" + maxPerFileSize); - } else { - LOG.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>> EMPTY PER FILE SIZE:"); } } @@ -72,14 +71,13 @@ public void init() throws IOException { meta = CatalogUtil.newTableMeta(plan.getStorageType()); } - fs = context.getOutputPath().getFileSystem(context.getConf()); openNewFile(writtenFileNum); } public void openNewFile(int suffixId) throws IOException { lastFileName = context.getOutputPath(); if (suffixId > 0) { - lastFileName = new Path(lastFileName, "_" + suffixId); + lastFileName = new Path(lastFileName + "_" + suffixId); } if (plan instanceof InsertNode) { @@ -97,8 +95,6 @@ public void openNewFile(int suffixId) throws IOException { appender.init(); } - long writtenSize = 0; - /* (non-Javadoc) * @see PhysicalExec#next() */ @@ -106,8 +102,9 @@ public void openNewFile(int suffixId) throws IOException { public Tuple next() throws IOException { while((tuple = child.next()) != null) { appender.addTuple(tuple); + writtenTupleSize += MemoryUtil.calculateMemorySize(tuple); - if (writtenSize > maxPerFileSize) { + if (writtenTupleSize > maxPerFileSize) { appender.close(); writtenFileNum++; @@ -117,10 +114,8 @@ public Tuple next() throws IOException { StatisticsUtil.aggregateTableStat(sumStats, appender.getStats()); } openNewFile(writtenFileNum); - LOG.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>> File Flushed" + maxPerFileSize); - writtenSize = 0; + writtenTupleSize = 0; } - writtenSize += MemoryUtil.calculateMemorySize(tuple); } return null; @@ -141,6 +136,7 @@ public void close() throws IOException { if (sumStats == null) { context.setResultStats(appender.getStats()); } else { + StatisticsUtil.aggregateTableStat(sumStats, appender.getStats()); context.setResultStats(sumStats); } if (context.getTaskId() != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 7a28d3843c..9b023fc3b9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -42,7 +42,7 @@ public class QueryContext extends KeyValueSet { public static final String OUTPUT_PARTITIONS = "tajo.query.output.partitions"; public static final String OUTPUT_OVERWRITE = "tajo.query.output.overwrite"; public static final String OUTPUT_AS_DIRECTORY = "tajo.query.output.asdirectory"; - public static final String OUTPUT_PER_FILE_SIZE = "tajo.query.output.perfile-size-mb"; + public static final String OUTPUT_PER_FILE_SIZE = "tajo.query.output.perfile-size-bytes"; public static final String TRUE_VALUE = "true"; public static final String FALSE_VALUE = "false"; From dd79f666d81875bf6a547478b76fc55b60f37d09 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 15 Jul 2014 21:31:11 +0900 Subject: [PATCH 04/13] Added estimatedwrittensize. --- .../physical/SortBasedColPartitionStoreExec.java | 3 +-- .../tajo/engine/planner/physical/StoreTableExec.java | 10 +++++++--- .../master/querymaster/QueryMasterManagerService.java | 4 +++- .../apache/tajo/storage/parquet/ParquetAppender.java | 2 +- .../parquet/InternalParquetRecordWriter.java | 4 ++-- .../tajo/storage/thirdparty/parquet/ParquetWriter.java | 4 ++++ 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java index c20ec3fd32..34ff4890aa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java @@ -163,9 +163,8 @@ public Tuple next() throws IOException { } appender.addTuple(tuple); - writtenTupleSize += MemoryUtil.calculateMemorySize(tuple); - if (writtenTupleSize > maxPerFileSize) { + if (maxPerFileSize <= appender.getEstimatedOutputSize()) { appender.close(); writtenFileNum++; StatisticsUtil.aggregateTableStat(aggregated, appender.getStats()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index c706ad2742..bafae9dd69 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -70,6 +70,12 @@ public void init() throws IOException { meta = CatalogUtil.newTableMeta(plan.getStorageType()); } + if (!(plan instanceof InsertNode)) { + String nullChar = context.getQueryContext().get(ConfVars.CSVFILE_NULL.varname, ConfVars.CSVFILE_NULL.defaultVal); + meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); + } + + openNewFile(writtenFileNum); } @@ -84,8 +90,6 @@ public void openNewFile(int suffixId) throws IOException { appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, createTableNode.getTableSchema(), lastFileName); } else { - String nullChar = context.getQueryContext().get(ConfVars.CSVFILE_NULL.varname, ConfVars.CSVFILE_NULL.defaultVal); - meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, lastFileName); } @@ -102,7 +106,7 @@ public Tuple next() throws IOException { while((tuple = child.next()) != null) { appender.addTuple(tuple); - if (maxPerFileSize > 0 && appender.getEstimatedOutputSize() > maxPerFileSize) { + if (maxPerFileSize <= appender.getEstimatedOutputSize()) { appender.close(); writtenFileNum++; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java index 826052d83a..3f860bfe7e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java @@ -204,7 +204,9 @@ public void done(RpcController controller, TajoWorkerProtocol.TaskCompletionRepo try { QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask( new QueryId(report.getId().getQueryUnitId().getExecutionBlockId().getQueryId())); - queryMasterTask.getEventHandler().handle(new TaskCompletionEvent(report)); + if (queryMasterTask != null) { + queryMasterTask.getEventHandler().handle(new TaskCompletionEvent(report)); + } done.run(TajoWorker.TRUE_PROTO); } catch (Exception e) { LOG.error(e.getMessage(), e); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java index 3d8ecc42ef..ff9e43c9b6 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java @@ -130,7 +130,7 @@ public void close() throws IOException { } public long getEstimatedOutputSize() throws IOException { - return writer. + return writer.getEstimatedWrittenSize(); } /** diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java b/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java index b5992ab9a7..7410d2bddb 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java @@ -139,8 +139,8 @@ private void checkBlockSizeReached() throws IOException { } } - public long getEstimatedWrittenSize() { - return store.memSize() + public long getEstimatedWrittenSize() throws IOException { + return w.getPos() + store.memSize(); } private void flushStore() diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetWriter.java b/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetWriter.java index 743d168a83..0447a47bd4 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetWriter.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetWriter.java @@ -210,6 +210,10 @@ public void write(T object) throws IOException { } } + public long getEstimatedWrittenSize() throws IOException { + return this.writer.getEstimatedWrittenSize(); + } + @Override public void close() throws IOException { try { From c006382a3b16973872d753c9a0e0150da1c0f687 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 17 Jul 2014 12:10:20 +0900 Subject: [PATCH 05/13] Reflect session variables to GlobalPlanner, Repartitioner, and PhysicalPlannerImpl. --- .../src/main/java/org/apache/tajo/conf/TajoConf.java | 8 ++++++-- .../tajo/engine/planner/PhysicalPlannerImpl.java | 12 ++++++++---- .../tajo/engine/planner/global/GlobalPlanner.java | 10 +++++++--- .../tajo/master/querymaster/Repartitioner.java | 9 +++++---- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 83ff9edb7f..2e647e8b1d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -448,8 +448,12 @@ public void setIntVar(ConfVars var, int val) { } public static long getLongVar(Configuration conf, ConfVars var) { - assert (var.valClass == Long.class); - return conf.getLong(var.varname, var.defaultLongVal); + assert (var.valClass == Long.class || var.valClass == Integer.class); + if (var.valClass == Integer.class) { + return conf.getInt(var.varname, var.defaultIntVal); + } else { + return conf.getLong(var.varname, var.defaultLongVal); + } } public static long getLongVar(Configuration conf, ConfVars var, long defaultVal) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index e271d3f586..e63d746d94 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -256,7 +256,8 @@ public boolean checkIfInMemoryInnerJoinIsPossible(TaskAttemptContext context, Lo throws IOException { String [] lineage = PlannerUtil.getRelationLineage(node); long volume = estimateSizeRecursive(context, lineage); - boolean inMemoryInnerJoinFlag = volume <= QueryContext.getLongVar(context.getQueryContext(), conf, ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD); + boolean inMemoryInnerJoinFlag = volume <= QueryContext.getLongVar(context.getQueryContext(), conf, + ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD); LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.", context.getTaskId().toString(), (left ? "Left" : "Right"), @@ -469,7 +470,8 @@ private PhysicalExec createBestLeftOuterJoinPlan(TaskAttemptContext context, Joi String [] rightLineage = PlannerUtil.getRelationLineage(plan.getRightChild()); long rightTableVolume = estimateSizeRecursive(context, rightLineage); - if (rightTableVolume < QueryContext.getLongVar(context.getQueryContext(), conf, ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)) { + if (rightTableVolume < QueryContext.getLongVar(context.getQueryContext(), conf, + ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)) { // we can implement left outer join using hash join, using the right operand as the build relation LOG.info("Left Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, leftExec, rightExec); @@ -487,7 +489,8 @@ private PhysicalExec createBestRightJoinPlan(TaskAttemptContext context, JoinNod // blocking, but merge join is blocking as well) String [] outerLineage4 = PlannerUtil.getRelationLineage(plan.getLeftChild()); long outerSize = estimateSizeRecursive(context, outerLineage4); - if (outerSize < QueryContext.getLongVar(context.getQueryContext(), conf, ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)){ + if (outerSize < QueryContext.getLongVar(context.getQueryContext(), conf, + ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)){ LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, rightExec, leftExec); } else { @@ -969,7 +972,8 @@ private PhysicalExec createBestAggregationPlan(TaskAttemptContext context, Group String [] outerLineage = PlannerUtil.getRelationLineage(groupbyNode.getChild()); long estimatedSize = estimateSizeRecursive(context, outerLineage); - final long threshold = QueryContext.getLongVar(context.getQueryContext(), conf, ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD); + final long threshold = QueryContext.getLongVar(context.getQueryContext(), conf, + ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD); // if the relation size is less than the threshold, // the hash aggregation will be used. diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 69ecd022cf..8bcdbf5d2d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -38,6 +38,7 @@ import org.apache.tajo.engine.planner.global.builder.DistinctGroupbyBuilder; import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InternalException; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; @@ -120,7 +121,8 @@ public void build(MasterPlan masterPlan) throws IOException, PlanningException { LogicalNode inputPlan = PlannerUtil.clone(masterPlan.getLogicalPlan(), masterPlan.getLogicalPlan().getRootBlock().getRoot()); - boolean autoBroadcast = conf.getBoolVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); + boolean autoBroadcast = QueryContext.getBoolVar(masterPlan.getContext(), conf, + TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); if (autoBroadcast) { // pre-visit the master plan in order to find tables to be broadcasted @@ -268,8 +270,10 @@ private ExecutionBlock buildJoinPlan(GlobalPlanContext context, JoinNode joinNod MasterPlan masterPlan = context.plan; ExecutionBlock currentBlock; - boolean autoBroadcast = conf.getBoolVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); - long broadcastThreshold = conf.getLongVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD); + boolean autoBroadcast = QueryContext.getBoolVar(context.getPlan().getContext(), conf, + TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); + long broadcastThreshold = QueryContext.getLongVar(context.getPlan().getContext(), conf, + TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD); // to check when the tajo.dist-query.join.broadcast.auto property is true if (autoBroadcast && joinNode.isCandidateBroadcast()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index 055e9a2056..e0204b36be 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -39,6 +39,7 @@ import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.utils.TupleUtil; import org.apache.tajo.exception.InternalException; import org.apache.tajo.ipc.TajoWorkerProtocol; @@ -373,8 +374,8 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster // Getting the desire number of join tasks according to the volumn // of a larger table int largerIdx = stats[0] >= stats[1] ? 0 : 1; - int desireJoinTaskVolumn = subQuery.getContext().getConf(). - getIntVar(ConfVars.DIST_QUERY_JOIN_TASK_VOLUME); + int desireJoinTaskVolumn = QueryContext.getIntVar(subQuery.getMasterPlan().getContext(), + subQuery.getContext().getConf(), ConfVars.DIST_QUERY_JOIN_TASK_VOLUME); // calculate the number of tasks according to the data size int mb = (int) Math.ceil((double) stats[largerIdx] / 1048576); @@ -758,8 +759,8 @@ public static void scheduleScatteredHashShuffleFetches(TaskSchedulerContext sche SubQuery subQuery, Map> intermediates, String tableName) { int i = 0; - int splitVolume = subQuery.getContext().getConf(). - getIntVar(ConfVars.DIST_QUERY_TABLE_PARTITION_VOLUME); + int splitVolume = QueryContext.getIntVar(subQuery.getMasterPlan().getContext(), + subQuery.getContext().getConf(), ConfVars.DIST_QUERY_TABLE_PARTITION_VOLUME); long sumNumBytes = 0L; Map> fetches = new HashMap>(); From 244318ccf8154c4b4e45dc24fdab23f85513e496 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 25 Jul 2014 10:27:08 +0900 Subject: [PATCH 06/13] Reset output rotating part. --- .../HashBasedColPartitionStoreExec.java | 10 --- .../SortBasedColPartitionStoreExec.java | 77 ++----------------- .../planner/physical/StoreTableExec.java | 68 +++------------- .../tajo/worker/TaskAttemptContext.java | 4 - .../org/apache/tajo/storage/Appender.java | 2 - .../org/apache/tajo/storage/FileAppender.java | 4 - .../tajo/storage/parquet/ParquetAppender.java | 4 - .../parquet/InternalParquetRecordWriter.java | 4 - .../thirdparty/parquet/ParquetWriter.java | 4 - 9 files changed, 16 insertions(+), 161 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java index b827c55ea9..6cef22e57a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java @@ -22,13 +22,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.planner.logical.InsertNode; import org.apache.tajo.engine.planner.logical.StoreTableNode; import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.StorageManagerFactory; import org.apache.tajo.storage.Tuple; import org.apache.tajo.worker.TaskAttemptContext; @@ -52,12 +48,6 @@ public HashBasedColPartitionStoreExec(TaskAttemptContext context, StoreTableNode } public void init() throws IOException { - if (plan instanceof InsertNode) { - String nullChar = context.getQueryContext().get(TajoConf.ConfVars.CSVFILE_NULL.varname, - TajoConf.ConfVars.CSVFILE_NULL.defaultVal); - meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); - } - super.init(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java index 34ff4890aa..d09e296706 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java @@ -23,17 +23,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.planner.logical.InsertNode; import org.apache.tajo.engine.planner.logical.StoreTableNode; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.*; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; @@ -54,25 +47,9 @@ public class SortBasedColPartitionStoreExec extends ColPartitionStoreExec { private Appender appender; private TableStats aggregated; - // for file rotating - private long maxPerFileSize = Long.MAX_VALUE; - private int writtenFileNum = 0; - private Path lastFileName; - private long writtenTupleSize = 0; - public SortBasedColPartitionStoreExec(TaskAttemptContext context, StoreTableNode plan, PhysicalExec child) throws IOException { super(context, plan, child); - - if (context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE) != null) { - maxPerFileSize = Long.valueOf(context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE)); - } - - if (plan instanceof InsertNode) { - String nullChar = context.getQueryContext().get(TajoConf.ConfVars.CSVFILE_NULL.varname, - TajoConf.ConfVars.CSVFILE_NULL.defaultVal); - meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); - } } public void init() throws IOException { @@ -82,41 +59,11 @@ public void init() throws IOException { aggregated = new TableStats(); } - private Appender getAppenderForNewPartition(String partition) throws IOException { - lastFileName = getDataFile(partition); - FileSystem fs = lastFileName.getFileSystem(context.getConf()); - - if (fs.exists(lastFileName.getParent())) { - LOG.info("Path " + lastFileName.getParent() + " already exists!"); - } else { - fs.mkdirs(lastFileName.getParent()); - LOG.info("Add subpartition path directory :" + lastFileName.getParent()); - } - - if (fs.exists(lastFileName)) { - LOG.info("File " + lastFileName + " already exists!"); - FileStatus status = fs.getFileStatus(lastFileName); - LOG.info("File size: " + status.getLen()); - } - - openNewFile(0); - + private Appender getAppender(String partition) throws IOException { + this.appender = makeAppender(partition); return appender; } - public void openNewFile(int suffixId) throws IOException { - Path actualFilePath = lastFileName; - if (suffixId > 0) { - actualFilePath = new Path(lastFileName + "_" + suffixId); - } - - appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, - outSchema, actualFilePath); - - appender.enableStats(); - appender.init(); - } - private void fillKeyTuple(Tuple inTuple, Tuple keyTuple) { for (int i = 0; i < keyIds.length; i++) { keyTuple.put(i, inTuple.get(keyIds[i])); @@ -145,33 +92,19 @@ public Tuple next() throws IOException { fillKeyTuple(tuple, currentKey); if (prevKey == null) { - appender = getAppenderForNewPartition(getSubdirectory(currentKey)); + appender = getAppender(getSubdirectory(currentKey)); prevKey = new VTuple(currentKey); } else { - - if (!prevKey.equals(currentKey)) { // new partition + if (!prevKey.equals(currentKey)) { appender.close(); StatisticsUtil.aggregateTableStat(aggregated, appender.getStats()); - appender = getAppenderForNewPartition(getSubdirectory(currentKey)); + appender = getAppender(getSubdirectory(currentKey)); prevKey = new VTuple(currentKey); - - // reset all states for file rotating - writtenTupleSize = 0; - writtenFileNum = 0; } } appender.addTuple(tuple); - - if (maxPerFileSize <= appender.getEstimatedOutputSize()) { - appender.close(); - writtenFileNum++; - StatisticsUtil.aggregateTableStat(aggregated, appender.getStats()); - - openNewFile(writtenFileNum); - writtenTupleSize = 0; - } } return null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index bafae9dd69..3a10eb1287 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -18,19 +18,15 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.statistics.StatisticsUtil; -import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.logical.InsertNode; import org.apache.tajo.engine.planner.logical.PersistentStoreNode; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.*; +import org.apache.tajo.storage.Appender; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.storage.StorageManagerFactory; +import org.apache.tajo.storage.Tuple; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; @@ -39,59 +35,34 @@ * This is a physical executor to store a table part into a specified storage. */ public class StoreTableExec extends UnaryPhysicalExec { - private static Log LOG = LogFactory.getLog(StoreTableExec.class); - private PersistentStoreNode plan; - private TableMeta meta; private Appender appender; private Tuple tuple; - private TableStats sumStats; - - // for file rotating - private long maxPerFileSize = Long.MAX_VALUE; - private int writtenFileNum = 0; - private Path lastFileName; public StoreTableExec(TaskAttemptContext context, PersistentStoreNode plan, PhysicalExec child) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema(), child); this.plan = plan; - - if (context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE) != null) { - maxPerFileSize = Long.valueOf(context.getQueryContext().get(QueryContext.OUTPUT_PER_FILE_SIZE)); - } } public void init() throws IOException { super.init(); + TableMeta meta; if (plan.hasOptions()) { meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions()); } else { meta = CatalogUtil.newTableMeta(plan.getStorageType()); } - if (!(plan instanceof InsertNode)) { - String nullChar = context.getQueryContext().get(ConfVars.CSVFILE_NULL.varname, ConfVars.CSVFILE_NULL.defaultVal); - meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); - } - - - openNewFile(writtenFileNum); - } - - public void openNewFile(int suffixId) throws IOException { - lastFileName = context.getOutputPath(); - if (suffixId > 0) { - lastFileName = new Path(lastFileName + "_" + suffixId); - } - if (plan instanceof InsertNode) { InsertNode createTableNode = (InsertNode) plan; appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, - createTableNode.getTableSchema(), lastFileName); + createTableNode.getTableSchema(), context.getOutputPath()); } else { + String nullChar = context.getQueryContext().get(ConfVars.CSVFILE_NULL.varname, ConfVars.CSVFILE_NULL.defaultVal); + meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, - lastFileName); + context.getOutputPath()); } appender.enableStats(); @@ -105,20 +76,8 @@ public void openNewFile(int suffixId) throws IOException { public Tuple next() throws IOException { while((tuple = child.next()) != null) { appender.addTuple(tuple); - - if (maxPerFileSize <= appender.getEstimatedOutputSize()) { - appender.close(); - writtenFileNum++; - - if (sumStats == null) { - sumStats = appender.getStats(); - } else { - StatisticsUtil.aggregateTableStat(sumStats, appender.getStats()); - } - openNewFile(writtenFileNum); - } } - + return null; } @@ -134,12 +93,7 @@ public void close() throws IOException { appender.flush(); appender.close(); // Collect statistics data - if (sumStats == null) { - context.setResultStats(appender.getStats()); - } else { - StatisticsUtil.aggregateTableStat(sumStats, appender.getStats()); - context.setResultStats(sumStats); - } + context.setResultStats(appender.getStats()); if (context.getTaskId() != null) { context.addShuffleFileOutput(0, context.getTaskId().toString()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index 05b9a3bfb6..250d4668b6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -113,10 +113,6 @@ public TajoConf getConf() { return this.conf; } - public String getConfig(String key) { - return queryContext.get(key) != null ? queryContext.get(key) : conf.get(key); - } - public TaskAttemptState getState() { return this.state; } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java index c5e96ac886..5b42cbd639 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java @@ -31,8 +31,6 @@ public interface Appender extends Closeable { void flush() throws IOException; - long getEstimatedOutputSize() throws IOException; - void close() throws IOException; void enableStats(); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java index 04278e9174..064841f77b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java @@ -57,9 +57,5 @@ public void enableStats() { this.enabledStats = true; } - public long getEstimatedOutputSize() throws IOException { - return getOffset(); - } - public abstract long getOffset() throws IOException; } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java index ff9e43c9b6..10b9331b48 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetAppender.java @@ -129,10 +129,6 @@ public void close() throws IOException { writer.close(); } - public long getEstimatedOutputSize() throws IOException { - return writer.getEstimatedWrittenSize(); - } - /** * If table statistics is enabled, retrieve the table statistics. * diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java b/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java index 7410d2bddb..8ce4b1cd8a 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/InternalParquetRecordWriter.java @@ -139,10 +139,6 @@ private void checkBlockSizeReached() throws IOException { } } - public long getEstimatedWrittenSize() throws IOException { - return w.getPos() + store.memSize(); - } - private void flushStore() throws IOException { LOG.info(format("Flushing mem store to file. allocated memory: %,d", store.allocatedSize())); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetWriter.java b/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetWriter.java index 0447a47bd4..743d168a83 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetWriter.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetWriter.java @@ -210,10 +210,6 @@ public void write(T object) throws IOException { } } - public long getEstimatedWrittenSize() throws IOException { - return this.writer.getEstimatedWrittenSize(); - } - @Override public void close() throws IOException { try { From 244b2614f0986248cec43bd0012bce8c489ea95f Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 25 Jul 2014 15:28:35 +0900 Subject: [PATCH 07/13] Add SessionVars and improve QueryContext. --- .../java/org/apache/tajo/InstantConfig.java | 25 ++++ .../java/org/apache/tajo/SessionVars.java | 100 +++++++++++++++ .../java/org/apache/tajo/conf/TajoConf.java | 8 +- .../tajo/engine/query/QueryContext.java | 120 +++++++++++------- .../engine/query/QueryUnitRequestImpl.java | 4 +- .../QueryMasterManagerService.java | 3 +- 6 files changed, 210 insertions(+), 50 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/InstantConfig.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/SessionVars.java diff --git a/tajo-common/src/main/java/org/apache/tajo/InstantConfig.java b/tajo-common/src/main/java/org/apache/tajo/InstantConfig.java new file mode 100644 index 0000000000..179477dbdd --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/InstantConfig.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +public interface InstantConfig { + public static final String PREFIX = "%"; + + public String key(); +} diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java new file mode 100644 index 0000000000..86e7311876 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import org.apache.tajo.annotation.Nullable; + +import static org.apache.tajo.SessionVars.VariableMode.*; +import static org.apache.tajo.conf.TajoConf.ConfVars; + +public enum SessionVars implements InstantConfig { + + USER_NAME(ConfVars.USERNAME, "user name", SERVER_SIDE_VAR), + + // Client Connection + CLIENT_SESSION_EXPIRY_TIME(ConfVars.CLIENT_SESSION_EXPIRY_TIME, "", DEFAULT), + + // Command line interface + CLI_MAX_COLUMN(ConfVars.CLI_MAX_COLUMN, "", DEFAULT), + CLI_PRINT_PAUSE_NUM_RECORDS(ConfVars.CLI_PRINT_PAUSE_NUM_RECORDS, "", DEFAULT), + CLI_PRINT_PAUSE(ConfVars.CLI_PRINT_PAUSE, "", DEFAULT), + CLI_PRINT_ERROR_TRACE(ConfVars.CLI_PRINT_ERROR_TRACE, "", DEFAULT), + CLI_OUTPUT_FORMATTER_CLASS(ConfVars.CLI_OUTPUT_FORMATTER_CLASS, "", DEFAULT), + CLI_NULL_CHAR(ConfVars.CLI_NULL_CHAR, "", DEFAULT), + + ON_ERROR_STOP(ConfVars.CLI_ERROR_STOP, "tsql will exist if an error occurs.", DEFAULT), + + // Timezone & Date + TZ(ConfVars.TIMEZONE, "Timezone", FROM_SHELL_ENV), + DATE_ORDER(ConfVars.DATE_ORDER, "YMD", FROM_SHELL_ENV), + + // Locales and Character set (reserved variables, which are currently not used. + LANG(null, "Language", FROM_SHELL_ENV), + LC_ALL(null, "String sort order", FROM_SHELL_ENV), + LC_COLLATE(null, "String sort order", FROM_SHELL_ENV), + LC_CTYPE(null, "Character classification (What is a letter? Its upper-case equivalent?)", FROM_SHELL_ENV), + LC_MESSAGES(null, "Language of messages", FROM_SHELL_ENV), + LC_MONETARY(null, "Formatting of currency amounts", FROM_SHELL_ENV), + LC_NUMERIC(null, "Formatting of numbers", FROM_SHELL_ENV), + LC_TIME(null, "Formatting of dates and times", FROM_SHELL_ENV), + + // Query and Optimization + OUTPUT_PER_FILE_SIZE(null, "", DEFAULT) + ; + + private final ConfVars key; + private final String description; + private final VariableMode mode; + + public static enum VariableMode { + DEFAULT, + SERVER_SIDE_VAR, + FROM_SHELL_ENV + } + + public static SessionVars get(String name) { + return Enum.valueOf(SessionVars.class, name.toUpperCase()); + } + + SessionVars(@Nullable ConfVars key, String description, VariableMode mode) { + this.key = key; + this.description = description; + this.mode = mode; + } + + public String key() { + return PREFIX + key.name(); + } + + public ConfVars getConfVars() { + return key; + } + + public Class getVarType() { + return key.valClass; + } + + public String getDescription() { + return description; + } + + public VariableMode getMode() { + return mode; + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 2e647e8b1d..8a46f664e5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -81,7 +81,7 @@ public static TimeZone getCurrentTimeZone() { try { if (CURRENT_TIMEZONE == null) { TajoConf tajoConf = new TajoConf(); - CURRENT_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.TAJO_TIMEZONE)); + CURRENT_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.TIMEZONE)); } return CURRENT_TIMEZONE; } finally { @@ -105,7 +105,7 @@ public static int getDateOrder() { try { if (DATE_ORDER < 0) { TajoConf tajoConf = new TajoConf(); - String dateOrder = tajoConf.getVar(ConfVars.TAJO_DATE_ORDER); + String dateOrder = tajoConf.getVar(ConfVars.DATE_ORDER); if ("YMD".equals(dateOrder)) { DATE_ORDER = DateTimeConstants.DATEORDER_YMD; } else if ("DMY".equals(dateOrder)) { @@ -329,8 +329,8 @@ public static enum ConfVars { CLI_ERROR_STOP("tajo.cli.error.stop", false), //TIME & DATE - TAJO_TIMEZONE("tajo.timezone", System.getProperty("user.timezone")), - TAJO_DATE_ORDER("tajo.date.order", "YMD"), + TIMEZONE("tajo.timezone", System.getProperty("user.timezone")), + DATE_ORDER("tajo.date.order", "YMD"), //PLANNER PLANNER_USE_FILTER_PUSHDOWN("tajo.planner.use.filter.pushdown", true), diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 9b023fc3b9..797f4a1bf1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -20,37 +20,64 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.InstantConfig; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.util.KeyValueSet; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; public class QueryContext extends KeyValueSet { - private static final Log LOG = LogFactory.getLog(QueryContext.class); - - public static final String COMMAND_TYPE = "tajo.query.command"; - public static final String STAGING_DIR = "tajo.query.staging_dir"; + public static enum QueryVars implements InstantConfig { + COMMAND_TYPE, + STAGING_DIR(), + OUTPUT_TABLE_NAME(), + OUTPUT_TABLE_PATH(), + OUTPUT_PARTITIONS(), + OUTPUT_OVERWRITE(), + OUTPUT_AS_DIRECTORY(), + OUTPUT_PER_FILE_SIZE(), + ; + + QueryVars() { + } - public static final String USER_NAME = "tajo.query.username"; + public String key() { + return PREFIX + name(); + } + } - public static final String OUTPUT_TABLE_NAME = "tajo.query.output.table"; - public static final String OUTPUT_TABLE_PATH = "tajo.query.output.path"; - public static final String OUTPUT_PARTITIONS = "tajo.query.output.partitions"; - public static final String OUTPUT_OVERWRITE = "tajo.query.output.overwrite"; - public static final String OUTPUT_AS_DIRECTORY = "tajo.query.output.asdirectory"; - public static final String OUTPUT_PER_FILE_SIZE = "tajo.query.output.perfile-size-bytes"; + private static final Log LOG = LogFactory.getLog(QueryContext.class); public static final String TRUE_VALUE = "true"; public static final String FALSE_VALUE = "false"; - public QueryContext() {} + private final TajoConf conf; - public QueryContext(KeyValueSetProto proto) { + public QueryContext() { + this(new TajoConf()); + } + + public QueryContext(final TajoConf conf) { + this.conf = conf; + } + + public QueryContext(final TajoConf conf, KeyValueSetProto proto) { super(proto); + this.conf = conf; + } + + public TajoConf getConf() { + return conf; + } + + public void put(QueryVars key, String val) { + put(key.key(), val); } public void put(TajoConf.ConfVars key, String value) { @@ -61,33 +88,50 @@ public String get(TajoConf.ConfVars key) { return get(key.varname); } + public String get(SessionVars key) { + return get(key.name()); + } + + public String get(QueryVars key) { + return get(key.key()); + } + public String get(String key) { return super.get(key); } + public void setBool(QueryVars key, boolean val) { + put(key.key(), val ? TRUE_VALUE : FALSE_VALUE); + } + public void setBool(String key, boolean val) { put(key, val ? TRUE_VALUE : FALSE_VALUE); } + public boolean getBool(QueryVars key) { + String strVal = get(key); + return strVal != null ? strVal.equalsIgnoreCase(TRUE_VALUE) : false; + } + public boolean getBool(String key) { String strVal = get(key); return strVal != null ? strVal.equalsIgnoreCase(TRUE_VALUE) : false; } public void setUser(String username) { - put(USER_NAME, username); + put(SessionVars.USER_NAME.getConfVars(), username); } public String getUser() { - return get(USER_NAME); + return get(SessionVars.USER_NAME); } public void setStagingDir(Path path) { - put(STAGING_DIR, path.toUri().toString()); + put(QueryVars.STAGING_DIR, path.toUri().toString()); } public Path getStagingDir() { - String strVal = get(STAGING_DIR); + String strVal = get(QueryVars.STAGING_DIR); return strVal != null ? new Path(strVal) : null; } @@ -97,7 +141,7 @@ public Path getStagingDir() { * This config is not set if a query has INSERT (OVERWRITE) INTO LOCATION '/path/..'. */ public boolean hasOutputTable() { - return get(OUTPUT_TABLE_NAME) != null; + return get(QueryVars.OUTPUT_TABLE_NAME) != null; } /** @@ -106,11 +150,11 @@ public boolean hasOutputTable() { * @param tableName The target table name */ public void setOutputTable(String tableName) { - put(OUTPUT_TABLE_NAME, tableName); + put(QueryVars.OUTPUT_TABLE_NAME, tableName); } public String getOutputTable() { - String strVal = get(OUTPUT_TABLE_NAME); + String strVal = get(QueryVars.OUTPUT_TABLE_NAME); return strVal != null ? strVal : null; } @@ -121,52 +165,48 @@ public String getOutputTable() { * @return */ public boolean hasOutputPath() { - return get(OUTPUT_TABLE_PATH) != null; + return get(QueryVars.OUTPUT_TABLE_PATH) != null; } public void setOutputPath(Path path) { - put(OUTPUT_TABLE_PATH, path.toUri().toString()); + put(QueryVars.OUTPUT_TABLE_PATH, path.toUri().toString()); } public Path getOutputPath() { - String strVal = get(OUTPUT_TABLE_PATH); + String strVal = get(QueryVars.OUTPUT_TABLE_PATH); return strVal != null ? new Path(strVal) : null; } public boolean hasPartition() { - return get(OUTPUT_PARTITIONS) != null; + return get(QueryVars.OUTPUT_PARTITIONS) != null; } public void setPartitionMethod(PartitionMethodDesc partitionMethodDesc) { - put(OUTPUT_PARTITIONS, partitionMethodDesc != null ? partitionMethodDesc.toJson() : null); + put(QueryVars.OUTPUT_PARTITIONS, partitionMethodDesc != null ? partitionMethodDesc.toJson() : null); } public PartitionMethodDesc getPartitionMethod() { - return PartitionMethodDesc.fromJson(get(OUTPUT_PARTITIONS)); + return PartitionMethodDesc.fromJson(get(QueryVars.OUTPUT_PARTITIONS)); } public void setOutputOverwrite() { - setBool(OUTPUT_OVERWRITE, true); + setBool(QueryVars.OUTPUT_OVERWRITE, true); } public boolean isOutputOverwrite() { - return getBool(OUTPUT_OVERWRITE); + return getBool(QueryVars.OUTPUT_OVERWRITE); } public void setFileOutput() { - setBool(OUTPUT_AS_DIRECTORY, true); - } - - public boolean isFileOutput() { - return getBool(OUTPUT_AS_DIRECTORY); + setBool(QueryVars.OUTPUT_AS_DIRECTORY, true); } public void setCommandType(NodeType nodeType) { - put(COMMAND_TYPE, nodeType.name()); + put(QueryVars.COMMAND_TYPE, nodeType.name()); } public NodeType getCommandType() { - String strVal = get(COMMAND_TYPE); + String strVal = get(QueryVars.COMMAND_TYPE); return strVal != null ? NodeType.valueOf(strVal) : null; } @@ -186,14 +226,6 @@ public boolean isInsert() { return getCommandType() == NodeType.INSERT; } - public void setHiveQueryMode() { - setBool("hive.query.mode", true); - } - - public boolean isHiveQueryMode() { - return getBool("hive.query.mode"); - } - public static boolean getBoolVar(QueryContext context, TajoConf conf, TajoConf.ConfVars key) { if (context.get(key.varname) != null) { return context.getBool(key.varname); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java index f1af2ff155..a1d1a29c08 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryUnitRequestImpl.java @@ -18,7 +18,9 @@ package org.apache.tajo.engine.query; +import org.apache.hadoop.conf.Configuration; import org.apache.tajo.QueryUnitAttemptId; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.ipc.TajoWorkerProtocol; @@ -193,7 +195,7 @@ public QueryContext getQueryContext() { if (!p.hasQueryContext()) { return null; } - this.queryContext = new QueryContext(p.getQueryContext()); + this.queryContext = new QueryContext(new TajoConf(), p.getQueryContext()); return this.queryContext; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java index 3f860bfe7e..ec975d8757 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java @@ -238,7 +238,8 @@ public void executeQuery(RpcController controller, LOG.info("Receive executeQuery request:" + queryId); queryMaster.handle(new QueryStartEvent(queryId, new Session(request.getSession()), - new QueryContext(request.getQueryContext()), request.getExprInJson().getValue(), + new QueryContext(workerContext.getQueryMaster().getContext().getConf(), + request.getQueryContext()), request.getExprInJson().getValue(), request.getLogicalPlanJson().getValue())); done.run(TajoWorker.TRUE_PROTO); } catch (Exception e) { From ed05c03448e9b49fa1280266d6b4fc38e473f314 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 25 Jul 2014 17:06:35 +0900 Subject: [PATCH 08/13] Improved QueryContext. --- .../java/org/apache/tajo/SessionVars.java | 22 +++++++++++- .../org/apache/tajo/util/KeyValueSet.java | 4 +++ .../tajo/engine/query/QueryContext.java | 35 +++++++++---------- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index 86e7311876..0cb0f0a0ca 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -18,8 +18,11 @@ package org.apache.tajo; +import com.google.common.collect.Maps; import org.apache.tajo.annotation.Nullable; +import java.util.Map; + import static org.apache.tajo.SessionVars.VariableMode.*; import static org.apache.tajo.conf.TajoConf.ConfVars; @@ -58,6 +61,15 @@ public enum SessionVars implements InstantConfig { OUTPUT_PER_FILE_SIZE(null, "", DEFAULT) ; + + private static Map SESSION_VARS = Maps.newHashMap(); + + static { + for (SessionVars var : SessionVars.values()) { + SESSION_VARS.put(var.key(), var); + } + } + private final ConfVars key; private final String description; private final VariableMode mode; @@ -68,8 +80,16 @@ public static enum VariableMode { FROM_SHELL_ENV } + public static boolean exists(String name) { + return SESSION_VARS.containsKey(name.toUpperCase()); + } + public static SessionVars get(String name) { - return Enum.valueOf(SessionVars.class, name.toUpperCase()); + if (exists(name)) { + return SESSION_VARS.get(name.toUpperCase()); + } else { + return null; + } } SessionVars(@Nullable ConfVars key, String description, VariableMode mode) { diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java index 4d1cee16d2..175c4ee0bc 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java @@ -63,6 +63,10 @@ public static KeyValueSet create(KeyValueSet keyValueSet) { public int size() { return keyVals.size(); } + + public boolean containsKey(String key) { + return this.keyVals.containsKey(key); + } public void put(String key, String val) { this.keyVals.put(key, val); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 797f4a1bf1..b6edddd0d6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -76,7 +76,7 @@ public TajoConf getConf() { return conf; } - public void put(QueryVars key, String val) { + public void put(InstantConfig key, String val) { put(key.key(), val); } @@ -84,31 +84,30 @@ public void put(TajoConf.ConfVars key, String value) { put(key.varname, value); } - public String get(TajoConf.ConfVars key) { - return get(key.varname); - } - - public String get(SessionVars key) { - return get(key.name()); + public String get(InstantConfig key) { + if (containsKey(key.key())) { + return get(key.key()); + } else if (key instanceof SessionVars) { + SessionVars sessionVar = SessionVars.get(key.key()); + return conf.getVar(sessionVar.getConfVars()); + } else { + throw new IllegalArgumentException("No Such a config key: " + key.key()); + } } - public String get(QueryVars key) { - return get(key.key()); + public String get(TajoConf.ConfVars key) { + return conf.getVar(key); } - public String get(String key) { - return super.get(key); - } - public void setBool(QueryVars key, boolean val) { + public void setBool(InstantConfig key, boolean val) { put(key.key(), val ? TRUE_VALUE : FALSE_VALUE); } - public void setBool(String key, boolean val) { - put(key, val ? TRUE_VALUE : FALSE_VALUE); - } - - public boolean getBool(QueryVars key) { + public boolean getBool(InstantConfig key) { + if (containsKey(key.key())) { + return get(key.key()) + } String strVal = get(key); return strVal != null ? strVal.equalsIgnoreCase(TRUE_VALUE) : false; } From 2becff67dafba948e37463ef840c2d96ce8c56a1 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 29 Jul 2014 19:16:01 +0900 Subject: [PATCH 09/13] Refactored QueryContext to take conf always. --- .../org/apache/tajo/catalog/TableMeta.java | 12 ++++ .../apache/tajo/catalog/TestKeyValueSet.java | 45 ++++++++++-- .../tajo/catalog/store/HCatalogStore.java | 4 +- .../java/org/apache/tajo/datum/NullDatum.java | 1 + .../org/apache/tajo/util/KeyValueSet.java | 6 +- .../engine/planner/PhysicalPlannerImpl.java | 13 ++-- .../engine/planner/global/GlobalPlanner.java | 10 +-- .../tajo/engine/query/QueryContext.java | 68 ++++++++++--------- .../org/apache/tajo/master/GlobalEngine.java | 2 +- .../master/querymaster/Repartitioner.java | 8 +-- .../tajo/worker/TaskAttemptContext.java | 1 + .../planner/global/TestBroadcastJoinPlan.java | 28 ++++---- .../planner/physical/TestBNLJoinExec.java | 4 +- .../planner/physical/TestBSTIndexExec.java | 2 +- .../physical/TestExternalSortExec.java | 2 +- .../physical/TestFullOuterHashJoinExec.java | 8 +-- .../physical/TestFullOuterMergeJoinExec.java | 12 ++-- .../physical/TestHashAntiJoinExec.java | 2 +- .../planner/physical/TestHashJoinExec.java | 5 +- .../physical/TestHashSemiJoinExec.java | 2 +- .../physical/TestLeftOuterHashJoinExec.java | 10 +-- .../physical/TestLeftOuterNLJoinExec.java | 10 +-- .../planner/physical/TestMergeJoinExec.java | 2 +- .../planner/physical/TestNLJoinExec.java | 4 +- .../planner/physical/TestPhysicalPlanner.java | 48 ++++++------- .../TestProgressExternalSortExec.java | 2 +- .../physical/TestRightOuterHashJoinExec.java | 6 +- .../physical/TestRightOuterMergeJoinExec.java | 12 ++-- .../engine/planner/physical/TestSortExec.java | 2 +- .../tajo/master/TestExecutionBlockCursor.java | 2 +- .../apache/tajo/master/TestGlobalPlanner.java | 2 +- .../worker/TestRangeRetrieverHandler.java | 4 +- .../java/org/apache/tajo/storage/CSVFile.java | 11 +-- .../apache/tajo/storage/avro/AvroUtil.java | 17 +++-- .../apache/tajo/storage/rcfile/RCFile.java | 10 +-- .../sequencefile/SequenceFileAppender.java | 10 +-- .../sequencefile/SequenceFileScanner.java | 3 +- .../apache/tajo/storage/v2/RCFileScanner.java | 2 +- 38 files changed, 225 insertions(+), 167 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java index 0a7d774464..2d95e6be09 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java @@ -81,6 +81,18 @@ public void putOption(String key, String val) { options.set(key, val); } + public boolean containsOption(String key) { + TableProtoOrBuilder p = viaProto ? proto : builder; + if (options != null) { + return this.options.containsKey(key); + } + if (!p.hasParams()) { + return false; + } + this.options = new KeyValueSet(p.getParams()); + return options.containsKey(key); + } + public String getOption(String key) { TableProtoOrBuilder p = viaProto ? proto : builder; if (options != null) { diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java index 14c553b214..b317ba4c85 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java @@ -31,11 +31,12 @@ public class TestKeyValueSet { @Test public final void testPutAndGetString() { KeyValueSet opts = new KeyValueSet(); - opts.set("key1", "val1"); - opts.set("key2", "val2"); + opts.set("k1", "v1"); + opts.set("k2", "v2"); + assertEquals("v1", opts.get("k1")); assertEquals("v2", opts.get("k2")); - assertEquals("v1", opts.get("v1")); + assertEquals("default", opts.get("k3", "default")); } @Test @@ -44,8 +45,42 @@ public final void testPutAndGetBool() { opts.setBool("k1", true); opts.setBool("k2", false); - assertEquals(false, opts.get("k1")); - assertEquals(true, opts.get("k2")); + assertEquals(true, opts.getBool("k1")); + assertEquals(false, opts.getBool("k2")); + assertEquals(true, opts.getBool("k3", true)); + } + + @Test + public final void testPutAndGetInt() { + KeyValueSet opts = new KeyValueSet(); + opts.setInt("k1", 1980); + opts.setInt("k2", 401); + + assertEquals(1980, opts.getInt("k1")); + assertEquals(401, opts.getInt("k2")); + assertEquals(2020, opts.getInt("k3", 2020)); + } + + @Test + public final void testPutAndGetLong() { + KeyValueSet opts = new KeyValueSet(); + opts.setLong("k1", 1980); + opts.setLong("k2", 401); + + assertEquals(1980, opts.getLong("k1")); + assertEquals(401, opts.getLong("k2")); + assertEquals(2020, opts.getLong("k3", 2020l)); + } + + @Test + public final void testPutAndGetFloat() { + KeyValueSet opts = new KeyValueSet(); + opts.setFloat("k1", 1980.4f); + opts.setFloat("k2", 401.150f); + + assertTrue(1980.4f == opts.getFloat("k1")); + assertTrue(401.150f == opts.getFloat("k2")); + assertTrue(3.14f == opts.getFloat("k3", 3.14f)); } @Test diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index 85c071b18a..c9046c545d 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -504,7 +504,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) StringEscapeUtils.unescapeJava(fieldDelimiter)); table.getParameters().remove(StorageConstants.CSVFILE_DELIMITER); - if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) { + if (tableDesc.getMeta().containsOption(StorageConstants.CSVFILE_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL))); table.getParameters().remove(StorageConstants.CSVFILE_NULL); @@ -534,7 +534,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName()); } - if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) { + if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL))); table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL); diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java index a4f79d7fec..532e7cd71b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java @@ -25,6 +25,7 @@ public class NullDatum extends Datum { private static NullDatum instance; + public static final String DEFAULT_TEXT = ""; private static final byte [] EMPTY_BYTES = new byte[0]; private static final DataType NULL_DATA_TYPE; diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java index e6b591f5c8..4cba082406 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java @@ -110,14 +110,14 @@ public void setBool(String key, boolean val) { set(key, val ? TRUE_STR : FALSE_STR); } - public Boolean getBool(String key, Boolean defaultVal) { + public boolean getBool(String key, Boolean defaultVal) { if (containsKey(key)) { String strVal = get(key, null); return strVal != null ? strVal.equalsIgnoreCase(TRUE_STR) : false; } else if (defaultVal != null) { return defaultVal; } else { - throw new IllegalArgumentException("No such a config key: " + key); + return false; } } @@ -180,7 +180,7 @@ public float getFloat(String key, Float defaultVal) { throw new IllegalArgumentException("No such a config key: " + key); } } else if (defaultVal != null) { - return defaultVal; + return defaultVal.floatValue(); } else { throw new IllegalArgumentException("No such a config key: " + key); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index e63d746d94..c5d018bca3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -256,8 +256,8 @@ public boolean checkIfInMemoryInnerJoinIsPossible(TaskAttemptContext context, Lo throws IOException { String [] lineage = PlannerUtil.getRelationLineage(node); long volume = estimateSizeRecursive(context, lineage); - boolean inMemoryInnerJoinFlag = volume <= QueryContext.getLongVar(context.getQueryContext(), conf, - ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD); + boolean inMemoryInnerJoinFlag = volume <= + context.getQueryContext().getLong(ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD); LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.", context.getTaskId().toString(), (left ? "Left" : "Right"), @@ -470,8 +470,7 @@ private PhysicalExec createBestLeftOuterJoinPlan(TaskAttemptContext context, Joi String [] rightLineage = PlannerUtil.getRelationLineage(plan.getRightChild()); long rightTableVolume = estimateSizeRecursive(context, rightLineage); - if (rightTableVolume < QueryContext.getLongVar(context.getQueryContext(), conf, - ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)) { + if (rightTableVolume < context.getQueryContext().getLong(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)) { // we can implement left outer join using hash join, using the right operand as the build relation LOG.info("Left Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, leftExec, rightExec); @@ -489,8 +488,7 @@ private PhysicalExec createBestRightJoinPlan(TaskAttemptContext context, JoinNod // blocking, but merge join is blocking as well) String [] outerLineage4 = PlannerUtil.getRelationLineage(plan.getLeftChild()); long outerSize = estimateSizeRecursive(context, outerLineage4); - if (outerSize < QueryContext.getLongVar(context.getQueryContext(), conf, - ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)){ + if (outerSize < context.getQueryContext().getLong(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)){ LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, rightExec, leftExec); } else { @@ -972,8 +970,7 @@ private PhysicalExec createBestAggregationPlan(TaskAttemptContext context, Group String [] outerLineage = PlannerUtil.getRelationLineage(groupbyNode.getChild()); long estimatedSize = estimateSizeRecursive(context, outerLineage); - final long threshold = QueryContext.getLongVar(context.getQueryContext(), conf, - ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD); + final long threshold = context.getQueryContext().getLong(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD); // if the relation size is less than the threshold, // the hash aggregation will be used. diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 844b8c5839..8c0df5dbe9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -121,10 +121,8 @@ public void build(MasterPlan masterPlan) throws IOException, PlanningException { LogicalNode inputPlan = PlannerUtil.clone(masterPlan.getLogicalPlan(), masterPlan.getLogicalPlan().getRootBlock().getRoot()); - boolean autoBroadcast = QueryContext.getBoolVar(masterPlan.getContext(), conf, - TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); + boolean autoBroadcast = masterPlan.getContext().getBool(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); if (autoBroadcast) { - // pre-visit the master plan in order to find tables to be broadcasted // this visiting does not make any execution block and change plan. BroadcastJoinMarkCandidateVisitor markCandidateVisitor = new BroadcastJoinMarkCandidateVisitor(); @@ -270,10 +268,8 @@ private ExecutionBlock buildJoinPlan(GlobalPlanContext context, JoinNode joinNod MasterPlan masterPlan = context.plan; ExecutionBlock currentBlock; - boolean autoBroadcast = QueryContext.getBoolVar(context.getPlan().getContext(), conf, - TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); - long broadcastThreshold = QueryContext.getLongVar(context.getPlan().getContext(), conf, - TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD); + boolean autoBroadcast = context.getPlan().getContext().getBool(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); + long broadcastThreshold = context.getPlan().getContext().getLong(ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD); // to check when the tajo.dist-query.join.broadcast.auto property is true if (autoBroadcast && joinNode.isCandidateBroadcast()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index b8ae20760d..b3d30bf5e1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -67,7 +67,7 @@ public static enum QueryVars implements ConfigKey { @Override public String keyname() { - return SESSION_PREFIX + name(); + return QUERY_CONF_PREFIX + name(); } @Override @@ -78,11 +78,7 @@ public ConfigType type() { private static final Log LOG = LogFactory.getLog(QueryContext.class); - private final TajoConf conf; - - public QueryContext() { - this(new TajoConf()); - } + private TajoConf conf; public QueryContext(final TajoConf conf) { this.conf = conf; @@ -93,6 +89,10 @@ public QueryContext(final TajoConf conf, KeyValueSetProto proto) { this.conf = conf; } + public void setConf(TajoConf conf) { + this.conf = conf; + } + public TajoConf getConf() { return conf; } @@ -173,12 +173,20 @@ public void put(ConfigKey key, String val) { set(key.keyname(), val); } + public static String getSessionKey(String key) { + return ConfigKey.SESSION_PREFIX + key; + } + + public static String getQueryKey(String key) { + return ConfigKey.QUERY_CONF_PREFIX + key; + } + public String get(ConfigKey key, String defaultVal) { switch (key.type()) { case QUERY: return get(key.keyname()); case SESSION: - return get(key.keyname(), conf.getVar(((SessionVars)key).getConfVars())); + return get(key.keyname(), conf.getVar(((SessionVars) key).getConfVars())); case SYSTEM: return conf.getVar((TajoConf.ConfVars) key); default: @@ -241,7 +249,7 @@ public String getOutputTable() { * @return */ public boolean hasOutputPath() { - return get(QueryVars.OUTPUT_TABLE_PATH) != null; + return containsKey(QueryVars.OUTPUT_TABLE_PATH); } public void setOutputPath(Path path) { @@ -254,7 +262,7 @@ public Path getOutputPath() { } public boolean hasPartition() { - return get(QueryVars.OUTPUT_PARTITIONS) != null; + return containsKey(QueryVars.OUTPUT_PARTITIONS); } public void setPartitionMethod(PartitionMethodDesc partitionMethodDesc) { @@ -277,6 +285,22 @@ public void setFileOutput() { setBool(QueryVars.OUTPUT_AS_DIRECTORY, true); } + public boolean containsKey(ConfigKey key) { + return containsKey(key.keyname()); + } + + public boolean equalKey(ConfigKey key, String another) { + if (containsKey(key)) { + return get(key).equals(another); + } else { + return false; + } + } + + public boolean assertCommandType(NodeType commandType) { + return equalKey(QueryVars.COMMAND_TYPE, commandType.name()); + } + public void setCommandType(NodeType nodeType) { put(QueryVars.COMMAND_TYPE, nodeType.name()); } @@ -291,7 +315,7 @@ public void setCreateTable() { } public boolean isCreateTable() { - return getCommandType() == NodeType.CREATE_TABLE; + return assertCommandType(NodeType.CREATE_TABLE); } public void setInsert() { @@ -299,29 +323,7 @@ public void setInsert() { } public boolean isInsert() { - return getCommandType() == NodeType.INSERT; - } - - public static boolean getBoolVar(QueryContext context, TajoConf conf, TajoConf.ConfVars key) { - if (context.get(key.varname) != null) { - return context.getBool(key.varname); - } else { - return conf.getBoolVar(key); - } - } - - public static Integer getIntVar(QueryContext context, TajoConf conf, TajoConf.ConfVars key) { - if (context.get(key.varname) != null) { - String val = context.get(key.varname); - try { - return Integer.valueOf(val); - } catch (NumberFormatException nfe) { - LOG.warn(nfe.getMessage()); - return conf.getIntVar(key); - } - } else { - return conf.getIntVar(key); - } + return assertCommandType(NodeType.INSERT); } public static Long getLongVar(QueryContext context, TajoConf conf, TajoConf.ConfVars key) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 73f3cf5854..b0cc40f50c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -116,7 +116,7 @@ public void stop() { public SubmitQueryResponse executeQuery(Session session, String query, boolean isJson) { LOG.info("Query: " + query); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(context.getConf()); queryContext.putAll(session.getAllVariables()); Expr planningContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index 664383aa6b..dd8d46f07c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -49,6 +49,7 @@ import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.TajoIdUtils; import org.apache.tajo.worker.FetchImpl; @@ -374,8 +375,7 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster // Getting the desire number of join tasks according to the volumn // of a larger table int largerIdx = stats[0] >= stats[1] ? 0 : 1; - int desireJoinTaskVolumn = QueryContext.getIntVar(subQuery.getMasterPlan().getContext(), - subQuery.getContext().getConf(), ConfVars.DIST_QUERY_JOIN_TASK_VOLUME); + int desireJoinTaskVolumn = subQuery.getMasterPlan().getContext().getInt(ConfVars.DIST_QUERY_JOIN_TASK_VOLUME); // calculate the number of tasks according to the data size int mb = (int) Math.ceil((double) stats[largerIdx] / 1048576); @@ -774,8 +774,8 @@ public static void scheduleScatteredHashShuffleFetches(TaskSchedulerContext sche SubQuery subQuery, Map> intermediates, String tableName) { int i = 0; - long splitVolume = ((long) 1048576) * QueryContext.getIntVar(subQuery.getMasterPlan().getContext(), - subQuery.getContext().getConf(), ConfVars.DIST_QUERY_TABLE_PARTITION_VOLUME); + long splitVolume = StorageUnit.MB * + subQuery.getMasterPlan().getContext().getLong(ConfVars.DIST_QUERY_TABLE_PARTITION_VOLUME); long sumNumBytes = 0L; Map> fetches = new HashMap>(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index 250d4668b6..0f791cb2da 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -81,6 +81,7 @@ public TaskAttemptContext(TajoConf conf, QueryContext queryContext, final QueryU final Path workDir) { this.conf = conf; this.queryContext = queryContext; + this.queryContext.setConf(conf); this.queryId = queryId; if (fragments != null) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index ec39609862..1cf5295296 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -188,7 +188,7 @@ public final void testBroadcastJoin() throws IOException, PlanningException { optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -249,7 +249,7 @@ public final void testBroadcastJoinAllSmallTables() throws IOException, Planning optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -313,7 +313,7 @@ public final void testNotBroadcastJoinTwoLargeTable() throws IOException, Planni optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -341,7 +341,7 @@ public final void testTwoBroadcastJoin() throws IOException, PlanningException { optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -391,7 +391,7 @@ public final void testNotBroadcastJoinSubquery() throws IOException, PlanningExc optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -432,7 +432,7 @@ public final void testBroadcastJoinSubquery() throws IOException, PlanningExcept optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -488,7 +488,7 @@ public final void testLeftOuterJoinCase1() throws IOException, PlanningException optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -542,7 +542,7 @@ public final void testLeftOuterJoinCase2() throws IOException, PlanningException optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -625,7 +625,7 @@ public final void testLeftOuterJoinCase3() throws IOException, PlanningException optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -708,7 +708,7 @@ public final void testLeftOuterJoinCase4() throws IOException, PlanningException optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -767,7 +767,7 @@ public final void testLeftOuterJoinCase5() throws IOException, PlanningException optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -820,7 +820,7 @@ public final void testLeftOuterJoinCase6() throws IOException, PlanningException optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -912,7 +912,7 @@ public final void testInnerLeftOuterJoinCase1() throws IOException, PlanningExce optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); @@ -977,7 +977,7 @@ public final void testBroadcastCasebyCase1() throws IOException, PlanningExcepti optimizer.optimize(plan); QueryId queryId = QueryIdFactory.newQueryId(System.currentTimeMillis(), 0); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); GlobalPlanner globalPlanner = new GlobalPlanner(conf, catalog); globalPlanner.build(masterPlan); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index d84796a2c8..6f42fd490f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -152,7 +152,7 @@ public final void testBNLCrossJoin() throws IOException, PlanningException { Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLCrossJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -189,7 +189,7 @@ public final void testBNLInnerJoin() throws IOException, PlanningException { enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index bfc3522770..3c1c46f4ac 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -169,7 +169,7 @@ public void testEqual() throws Exception { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEqual"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); Expr expr = analyzer.parse(QUERY); LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 1ce5b5bef1..ba269c9dd2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -121,7 +121,7 @@ public final void testNext() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 1a8a90e503..8345bd703d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -270,7 +270,7 @@ public final void testFullOuterHashJoinExec0() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -307,7 +307,7 @@ public final void testFullOuterHashJoinExec1() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuter_HashJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -343,7 +343,7 @@ public final void testFullOuterHashJoinExec2() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -381,7 +381,7 @@ public final void testFullOuterHashJoinExec3() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 50e5906740..2790fb747e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -316,7 +316,7 @@ public final void testFullOuterMergeJoin0() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -353,7 +353,7 @@ public final void testFullOuterMergeJoin1() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -390,7 +390,7 @@ public final void testFullOuterMergeJoin2() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -428,7 +428,7 @@ public final void testFullOuterMergeJoin3() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -469,7 +469,7 @@ public final void testFullOuterMergeJoin4() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -508,7 +508,7 @@ public final void testFullOuterMergeJoin5() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin5"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 794ca79654..bfb66f26ee 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -154,7 +154,7 @@ public final void testHashAntiJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashAntiJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index 0e0753618a..d41468cb0b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -152,7 +152,7 @@ public final void testHashInnerJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -195,12 +195,13 @@ public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, P FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); TajoConf localConf = new TajoConf(conf); localConf.setLongVar(TajoConf.ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD, 100l); + ctx.getQueryContext().setConf(localConf); PhysicalPlannerImpl phyPlanner = new PhysicalPlannerImpl(localConf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 835260f9eb..c9c2b06374 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -158,7 +158,7 @@ public final void testHashSemiJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashSemiJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index bb25875004..782f56fecb 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -272,7 +272,7 @@ public final void testLeftOuterHashJoinExec0() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterHashJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -303,7 +303,7 @@ public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[1]); @@ -344,7 +344,7 @@ public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[2]); @@ -386,7 +386,7 @@ public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[3]); @@ -428,7 +428,7 @@ public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[4]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index e935f578a9..634216d558 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -261,7 +261,7 @@ public final void testLeftOuterNLJoinExec0() throws IOException, PlanningExcepti FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterNLJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); @@ -303,7 +303,7 @@ public final void testLeftOuterNLJoinExec1() throws IOException, PlanningExcepti Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[1]); @@ -347,7 +347,7 @@ public final void testLeftOuter_NLJoinExec2() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[2]); @@ -392,7 +392,7 @@ public final void testLeftOuter_NLJoinExec3() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[3]); @@ -436,7 +436,7 @@ public final void testLeftOuter_NLJoinExec4() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[4]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index a4449fa653..0fec28245a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -166,7 +166,7 @@ public final void testMergeInnerJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testMergeInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 019929daab..7605e6596b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -152,7 +152,7 @@ public final void testNLCrossJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLCrossJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); @@ -181,7 +181,7 @@ public final void testNLInnerJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[1]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index a823d2bb9b..128226151e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -201,7 +201,7 @@ public final void testCreateScanPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -232,7 +232,7 @@ public final void testCreateScanWithFilterPlan() throws IOException, PlanningExc FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanWithFilterPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -261,7 +261,7 @@ public final void testGroupByPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -293,7 +293,7 @@ public final void testHashGroupByPlanWithALLField() throws IOException, Planning Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir( "target/test-data/testHashGroupByPlanWithALLField"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -322,7 +322,7 @@ public final void testSortGroupByPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortGroupByPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[]{frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); @@ -382,7 +382,7 @@ public final void testStorePlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -424,7 +424,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlanWithRCFile"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -465,7 +465,7 @@ public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOExcep FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -492,7 +492,7 @@ public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOExc FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); @@ -516,7 +516,7 @@ public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOExc FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); @@ -533,7 +533,7 @@ public final void testPartitionedStorePlan() throws IOException, PlanningExcepti Integer.MAX_VALUE); QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testPartitionedStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), id, new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[7]); @@ -594,7 +594,7 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() Path workDir = CommonTestingUtil.getTestDir( "target/test-data/testPartitionedStorePlanWithEmptyGroupingSet"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), id, new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[14]); @@ -648,7 +648,7 @@ public final void testAggregationFunction() throws IOException, PlanningExceptio FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testAggregationFunction"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -679,7 +679,7 @@ public final void testCountFunction() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCountFunction"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -707,7 +707,7 @@ public final void testGroupByWithNullValue() throws IOException, PlanningExcepti FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByWithNullValue"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -732,7 +732,7 @@ public final void testUnionPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testUnionPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -760,7 +760,7 @@ public final void testUnionPlan() throws IOException, PlanningException { @Test public final void testEvalExpr() throws IOException, PlanningException { Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEvalExpr"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { }, workDir); Expr expr = analyzer.parse(QUERIES[12]); @@ -797,7 +797,7 @@ public final void testCreateIndex() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); Expr context = analyzer.parse(createIndexStmt[0]); @@ -825,7 +825,7 @@ public final void testDuplicateEliminate() throws IOException, PlanningException Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testDuplicateEliminate"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); @@ -859,7 +859,7 @@ public final void testIndexedStoreExec() throws IOException, PlanningException { employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testIndexedStoreExec"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); @@ -961,7 +961,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.IN_MEMORY_SORT); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); @@ -983,7 +983,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { enforcer = new Enforcer(); enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.MERGE_SORT); - ctx = new TaskAttemptContext(conf, new QueryContext(), + ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); @@ -1011,7 +1011,7 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceHashAggregation(groupByNode.getPID()); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); @@ -1033,7 +1033,7 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { enforcer = new Enforcer(); enforcer.enforceSortAggregation(groupByNode.getPID(), null); - ctx = new TaskAttemptContext(conf, new QueryContext(), + ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index ed6cb4efc3..d779494d2a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -139,7 +139,7 @@ private void testProgress(int sortBufferBytesNum) throws Exception { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index de90f70f00..251b610a97 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -237,7 +237,7 @@ public final void testRightOuter_HashJoinExec0() throws IOException, PlanningExc FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); @@ -278,7 +278,7 @@ public final void testRightOuter_HashJoinExec1() throws IOException, PlanningExc FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[1]); @@ -319,7 +319,7 @@ public final void testRightOuter_HashJoinExec2() throws IOException, PlanningExc FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[2]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index d971073222..d71b6fdeee 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -315,7 +315,7 @@ public final void testRightOuterMergeJoin0() throws IOException, PlanningExcepti FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -352,7 +352,7 @@ public final void testRightOuter_MergeJoin1() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -388,7 +388,7 @@ public final void testRightOuterMergeJoin2() throws IOException, PlanningExcepti FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -424,7 +424,7 @@ public final void testRightOuter_MergeJoin3() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -462,7 +462,7 @@ public final void testRightOuter_MergeJoin4() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -499,7 +499,7 @@ public final void testRightOuterMergeJoin5() throws IOException, PlanningExcepti Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin5"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index da6fb34538..413ee41b9d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -110,7 +110,7 @@ public static void setUp() throws Exception { public final void testNext() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestSortExec"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility .newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 4b76cb8804..6a1a3cafca 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -109,7 +109,7 @@ public void testNextBlock() throws Exception { "join part on p_partkey = ps_partkey and p_type like '%BRASS' and p_size = 15"); LogicalPlan logicalPlan = logicalPlanner.createPlan(LocalTajoTestingUtility.createDummySession(), context); optimizer.optimize(logicalPlan); - QueryContext queryContext = new QueryContext(); + QueryContext queryContext = new QueryContext(conf); MasterPlan plan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), queryContext, logicalPlan); planner.build(plan); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java index 0ce77464e2..2d482ebe18 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java @@ -114,7 +114,7 @@ private MasterPlan buildPlan(String sql) throws PlanningException, IOException { Expr expr = sqlAnalyzer.parse(sql); LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); optimizer.optimize(plan); - QueryContext context = new QueryContext(); + QueryContext context = new QueryContext(util.getConfiguration()); MasterPlan masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), context, plan); globalPlanner.build(masterPlan); return masterPlan; diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index 53757497e0..66088bf37d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -143,7 +143,7 @@ public void testGet() throws Exception { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tableDir, Integer.MAX_VALUE); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] {frags[0]}, testDir); ctx.setEnforcer(new Enforcer()); @@ -267,7 +267,7 @@ public void testGetFromDescendingOrder() throws Exception { FileFragment[] frags = sm.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); TaskAttemptContext - ctx = new TaskAttemptContext(conf, new QueryContext(), + ctx = new TaskAttemptContext(conf, new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] {frags[0]}, testDir); ctx.setEnforcer(new Enforcer()); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java index 8e26ec6110..211379467e 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java @@ -84,7 +84,9 @@ public CSVAppender(Configuration conf, final Schema schema, final TableMeta meta this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0); this.columnNum = schema.size(); - String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.CSVFILE_NULL)); + + String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.CSVFILE_NULL, + NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); } else { @@ -107,8 +109,8 @@ public void init() throws IOException { isShuffle = false; } - String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); - if(!StringUtils.isEmpty(codecName)){ + if(this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) { + String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); codecFactory = new CompressionCodecFactory(conf); codec = codecFactory.getCodecByClassName(codecName); compressor = CodecPool.getCompressor(codec); @@ -262,7 +264,8 @@ public CSVScanner(Configuration conf, final Schema schema, final TableMeta meta, String delim = meta.getOption(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); this.delimiter = StringEscapeUtils.unescapeJava(delim).charAt(0); - String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.CSVFILE_NULL)); + String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.CSVFILE_NULL, + NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); } else { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java index 962c63d575..c15d20b5e1 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/avro/AvroUtil.java @@ -31,15 +31,20 @@ public class AvroUtil { public static Schema getAvroSchema(TableMeta meta, Configuration conf) throws IOException { - String schemaLiteral = meta.getOption(StorageConstants.AVRO_SCHEMA_LITERAL); - String schemaUrl = meta.getOption(StorageConstants.AVRO_SCHEMA_URL); - if (schemaLiteral == null && schemaUrl == null) { + + + boolean isSchemaLiteral = meta.containsOption(StorageConstants.AVRO_SCHEMA_LITERAL); + boolean isSchemaUrl = meta.containsOption(StorageConstants.AVRO_SCHEMA_URL); + if (!isSchemaLiteral && !isSchemaUrl) { throw new RuntimeException("No Avro schema for table."); } - if (schemaLiteral != null) { - return new Schema.Parser().parse(schemaLiteral); + if (isSchemaLiteral) { + String schema = meta.getOption(StorageConstants.AVRO_SCHEMA_LITERAL); + return new Schema.Parser().parse(schema); } - Path schemaPath = new Path(schemaUrl); + + String schemaURL = meta.getOption(StorageConstants.AVRO_SCHEMA_URL); + Path schemaPath = new Path(schemaURL); FileSystem fs = schemaPath.getFileSystem(conf); FSDataInputStream inputStream = fs.open(schemaPath); return new Schema.Parser().parse(inputStream); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java index 78498c750a..e5507ad81f 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java @@ -733,8 +733,8 @@ public void init() throws IOException { isShuffle = false; } - String codecClassname = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); - if (!StringUtils.isEmpty(codecClassname)) { + if (this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) { + String codecClassname = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); try { Class codecClass = conf.getClassByName( codecClassname).asSubclass(CompressionCodec.class); @@ -745,7 +745,8 @@ public void init() throws IOException { } } - String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.RCFILE_NULL)); + String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.RCFILE_NULL, + NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); } else { @@ -1193,7 +1194,8 @@ public void init() throws IOException { rowId = new LongWritable(); readBytes = 0; - String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.RCFILE_NULL)); + String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.RCFILE_NULL, + NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); } else { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java index b150a9a340..9935def5d0 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileAppender.java @@ -99,7 +99,8 @@ public void init() throws IOException { this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0); this.columnNum = schema.size(); - String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL)); + String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL, + NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); } else { @@ -110,8 +111,8 @@ public void init() throws IOException { throw new FileNotFoundException(path.toString()); } - String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); - if(!StringUtils.isEmpty(codecName)){ + if(this.meta.containsOption(StorageConstants.COMPRESSION_CODEC)) { + String codecName = this.meta.getOption(StorageConstants.COMPRESSION_CODEC); codecFactory = new CompressionCodecFactory(conf); codec = codecFactory.getCodecByClassName(codecName); } else { @@ -121,7 +122,8 @@ public void init() throws IOException { } try { - String serdeClass = this.meta.getOption(StorageConstants.SEQUENCEFILE_SERDE, TextSerializerDeserializer.class.getName()); + String serdeClass = this.meta.getOption(StorageConstants.SEQUENCEFILE_SERDE, + TextSerializerDeserializer.class.getName()); serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance(); } catch (Exception e) { LOG.error(e.getMessage(), e); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java index 32d1d57688..362c60c234 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java @@ -87,7 +87,8 @@ public void init() throws IOException { reader = new SequenceFile.Reader(fs, fragment.getPath(), conf); - String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL)); + String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.SEQUENCEFILE_NULL, + NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); } else { diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java index 5d2f861f8a..c8e19dda68 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java @@ -70,7 +70,7 @@ public RCFileScanner(final Configuration conf, final Schema schema, final TableM key = new LongWritable(); column = new BytesRefArrayWritable(); - String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(NULL)); + String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(NULL, NullDatum.DEFAULT_TEXT)); if (StringUtils.isEmpty(nullCharacters)) { nullChars = NullDatum.get().asTextBytes(); } else { From 59861210fe4f16b50b076fd0b0456d6f7bd8a145 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 29 Jul 2014 19:35:55 +0900 Subject: [PATCH 10/13] Refactored TaskAttemptContext to not take TajoConf. --- .../org/apache/tajo/master/GlobalEngine.java | 2 +- .../java/org/apache/tajo/worker/Task.java | 2 +- .../tajo/worker/TaskAttemptContext.java | 11 ++--- .../planner/physical/TestBNLJoinExec.java | 4 +- .../planner/physical/TestBSTIndexExec.java | 2 +- .../physical/TestExternalSortExec.java | 2 +- .../physical/TestFullOuterHashJoinExec.java | 8 ++-- .../physical/TestFullOuterMergeJoinExec.java | 12 ++--- .../physical/TestHashAntiJoinExec.java | 2 +- .../planner/physical/TestHashJoinExec.java | 4 +- .../physical/TestHashSemiJoinExec.java | 2 +- .../physical/TestLeftOuterHashJoinExec.java | 10 ++-- .../physical/TestLeftOuterNLJoinExec.java | 10 ++-- .../planner/physical/TestMergeJoinExec.java | 2 +- .../planner/physical/TestNLJoinExec.java | 4 +- .../planner/physical/TestPhysicalPlanner.java | 48 +++++++++---------- .../TestProgressExternalSortExec.java | 2 +- .../physical/TestRightOuterHashJoinExec.java | 6 +-- .../physical/TestRightOuterMergeJoinExec.java | 12 ++--- .../engine/planner/physical/TestSortExec.java | 2 +- .../worker/TestRangeRetrieverHandler.java | 4 +- 21 files changed, 74 insertions(+), 77 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index b0cc40f50c..a5345de490 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -310,7 +310,7 @@ private void insertNonFromQuery(QueryContext queryContext, InsertNode insertNode } TaskAttemptContext taskAttemptContext = - new TaskAttemptContext(context.getConf(), queryContext, null, (CatalogProtos.FragmentProto[]) null, stagingDir); + new TaskAttemptContext(queryContext, null, (CatalogProtos.FragmentProto[]) null, stagingDir); taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000")); EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild()); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 73fb8cd555..eb769260a7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -148,7 +148,7 @@ public Task(QueryUnitAttemptId taskId, this.taskDir = StorageUtil.concatPath(taskRunnerContext.getBaseDir(), taskId.getQueryUnitId().getId() + "_" + taskId.getId()); - this.context = new TaskAttemptContext(systemConf, queryContext, taskId, + this.context = new TaskAttemptContext(queryContext, taskId, request.getFragments().toArray(new FragmentProto[request.getFragments().size()]), taskDir); this.context.setDataChannel(request.getDataChannel()); this.context.setEnforcer(request.getEnforcer()); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index 0f791cb2da..211f9535ee 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -50,7 +50,6 @@ */ public class TaskAttemptContext { private static final Log LOG = LogFactory.getLog(TaskAttemptContext.class); - private final TajoConf conf; private final Map> fragmentMap = Maps.newHashMap(); private TaskAttemptState state; @@ -76,12 +75,10 @@ public class TaskAttemptContext { /** a output volume for each partition */ private Map partitionOutputVolume; - public TaskAttemptContext(TajoConf conf, QueryContext queryContext, final QueryUnitAttemptId queryId, + public TaskAttemptContext(final QueryContext queryContext, final QueryUnitAttemptId queryId, final FragmentProto[] fragments, final Path workDir) { - this.conf = conf; this.queryContext = queryContext; - this.queryContext.setConf(conf); this.queryId = queryId; if (fragments != null) { @@ -105,13 +102,13 @@ public TaskAttemptContext(TajoConf conf, QueryContext queryContext, final QueryU } @VisibleForTesting - public TaskAttemptContext(TajoConf conf, QueryContext queryContext, final QueryUnitAttemptId queryId, + public TaskAttemptContext(final QueryContext queryContext, final QueryUnitAttemptId queryId, final Fragment [] fragments, final Path workDir) { - this(conf, queryContext, queryId, FragmentConvertor.toFragmentProtoArray(fragments), workDir); + this(queryContext, queryId, FragmentConvertor.toFragmentProtoArray(fragments), workDir); } public TajoConf getConf() { - return this.conf; + return queryContext.getConf(); } public TaskAttemptState getState() { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index 6f42fd490f..43caec8506 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -152,7 +152,7 @@ public final void testBNLCrossJoin() throws IOException, PlanningException { Integer.MAX_VALUE); FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLCrossJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -189,7 +189,7 @@ public final void testBNLInnerJoin() throws IOException, PlanningException { enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index 3c1c46f4ac..e7c10a137d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -169,7 +169,7 @@ public void testEqual() throws Exception { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEqual"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); Expr expr = analyzer.parse(QUERY); LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index ba269c9dd2..9236c97886 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -121,7 +121,7 @@ public final void testNext() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 8345bd703d..8addcdb7f0 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -270,7 +270,7 @@ public final void testFullOuterHashJoinExec0() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -307,7 +307,7 @@ public final void testFullOuterHashJoinExec1() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuter_HashJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -343,7 +343,7 @@ public final void testFullOuterHashJoinExec2() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -381,7 +381,7 @@ public final void testFullOuterHashJoinExec3() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 2790fb747e..2b137cc9f5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -316,7 +316,7 @@ public final void testFullOuterMergeJoin0() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -353,7 +353,7 @@ public final void testFullOuterMergeJoin1() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -390,7 +390,7 @@ public final void testFullOuterMergeJoin2() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -428,7 +428,7 @@ public final void testFullOuterMergeJoin3() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -469,7 +469,7 @@ public final void testFullOuterMergeJoin4() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -508,7 +508,7 @@ public final void testFullOuterMergeJoin5() throws IOException, PlanningExceptio FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin5"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index bfb66f26ee..d4c3c309f0 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -154,7 +154,7 @@ public final void testHashAntiJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashAntiJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index d41468cb0b..8364c4120c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -152,7 +152,7 @@ public final void testHashInnerJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -195,7 +195,7 @@ public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, P FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index c9c2b06374..f13c11625b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -158,7 +158,7 @@ public final void testHashSemiJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashSemiJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 782f56fecb..96fb83add2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -272,7 +272,7 @@ public final void testLeftOuterHashJoinExec0() throws IOException, PlanningExcep FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterHashJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -303,7 +303,7 @@ public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[1]); @@ -344,7 +344,7 @@ public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[2]); @@ -386,7 +386,7 @@ public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[3]); @@ -428,7 +428,7 @@ public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningExce FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_HashJoinExec4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[4]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index 634216d558..66aab0ad0c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -261,7 +261,7 @@ public final void testLeftOuterNLJoinExec0() throws IOException, PlanningExcepti FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterNLJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); @@ -303,7 +303,7 @@ public final void testLeftOuterNLJoinExec1() throws IOException, PlanningExcepti Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[1]); @@ -347,7 +347,7 @@ public final void testLeftOuter_NLJoinExec2() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[2]); @@ -392,7 +392,7 @@ public final void testLeftOuter_NLJoinExec3() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[3]); @@ -436,7 +436,7 @@ public final void testLeftOuter_NLJoinExec4() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuter_NLJoinExec4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[4]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index 0fec28245a..97820b39e3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -166,7 +166,7 @@ public final void testMergeInnerJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testMergeInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 7605e6596b..397425970d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -152,7 +152,7 @@ public final void testNLCrossJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLCrossJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); @@ -181,7 +181,7 @@ public final void testNLInnerJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testNLInnerJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[1]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 128226151e..a890164ccf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -201,7 +201,7 @@ public final void testCreateScanPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -232,7 +232,7 @@ public final void testCreateScanWithFilterPlan() throws IOException, PlanningExc FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanWithFilterPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -261,7 +261,7 @@ public final void testGroupByPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -293,7 +293,7 @@ public final void testHashGroupByPlanWithALLField() throws IOException, Planning Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir( "target/test-data/testHashGroupByPlanWithALLField"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -322,7 +322,7 @@ public final void testSortGroupByPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortGroupByPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[]{frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); @@ -382,7 +382,7 @@ public final void testStorePlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -424,7 +424,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlanWithRCFile"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -465,7 +465,7 @@ public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOExcep FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -492,7 +492,7 @@ public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOExc FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); @@ -516,7 +516,7 @@ public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOExc FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); @@ -533,7 +533,7 @@ public final void testPartitionedStorePlan() throws IOException, PlanningExcepti Integer.MAX_VALUE); QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testPartitionedStorePlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), id, new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[7]); @@ -594,7 +594,7 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() Path workDir = CommonTestingUtil.getTestDir( "target/test-data/testPartitionedStorePlanWithEmptyGroupingSet"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), id, new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[14]); @@ -648,7 +648,7 @@ public final void testAggregationFunction() throws IOException, PlanningExceptio FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testAggregationFunction"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -679,7 +679,7 @@ public final void testCountFunction() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCountFunction"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -707,7 +707,7 @@ public final void testGroupByWithNullValue() throws IOException, PlanningExcepti FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByWithNullValue"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -732,7 +732,7 @@ public final void testUnionPlan() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testUnionPlan"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); @@ -760,7 +760,7 @@ public final void testUnionPlan() throws IOException, PlanningException { @Test public final void testEvalExpr() throws IOException, PlanningException { Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEvalExpr"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { }, workDir); Expr expr = analyzer.parse(QUERIES[12]); @@ -797,7 +797,7 @@ public final void testCreateIndex() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); Expr context = analyzer.parse(createIndexStmt[0]); @@ -825,7 +825,7 @@ public final void testDuplicateEliminate() throws IOException, PlanningException Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testDuplicateEliminate"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); @@ -859,7 +859,7 @@ public final void testIndexedStoreExec() throws IOException, PlanningException { employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testIndexedStoreExec"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); @@ -961,7 +961,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.IN_MEMORY_SORT); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); @@ -983,7 +983,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { enforcer = new Enforcer(); enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.MERGE_SORT); - ctx = new TaskAttemptContext(conf, new QueryContext(conf), + ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); @@ -1011,7 +1011,7 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { Enforcer enforcer = new Enforcer(); enforcer.enforceHashAggregation(groupByNode.getPID()); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); @@ -1033,7 +1033,7 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { enforcer = new Enforcer(); enforcer.enforceSortAggregation(groupByNode.getPID(), null); - ctx = new TaskAttemptContext(conf, new QueryContext(conf), + ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index d779494d2a..ef5fe0e1c3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -139,7 +139,7 @@ private void testProgress(int sortBufferBytesNum) throws Exception { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 251b610a97..46b0a8eea2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -237,7 +237,7 @@ public final void testRightOuter_HashJoinExec0() throws IOException, PlanningExc FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); @@ -278,7 +278,7 @@ public final void testRightOuter_HashJoinExec1() throws IOException, PlanningExc FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[1]); @@ -319,7 +319,7 @@ public final void testRightOuter_HashJoinExec2() throws IOException, PlanningExc FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestRightOuter_HashJoinExec2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[2]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index d71b6fdeee..49a9247b57 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -315,7 +315,7 @@ public final void testRightOuterMergeJoin0() throws IOException, PlanningExcepti FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin0"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -352,7 +352,7 @@ public final void testRightOuter_MergeJoin1() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin1"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -388,7 +388,7 @@ public final void testRightOuterMergeJoin2() throws IOException, PlanningExcepti FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin2"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -424,7 +424,7 @@ public final void testRightOuter_MergeJoin3() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin3"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -462,7 +462,7 @@ public final void testRightOuter_MergeJoin4() throws IOException, PlanningExcept FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin4"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); @@ -499,7 +499,7 @@ public final void testRightOuterMergeJoin5() throws IOException, PlanningExcepti Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin5"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 413ee41b9d..48317a75ef 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -110,7 +110,7 @@ public static void setUp() throws Exception { public final void testNext() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestSortExec"); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility .newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index 66088bf37d..875527f87a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -143,7 +143,7 @@ public void testGet() throws Exception { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tableDir, Integer.MAX_VALUE); - TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] {frags[0]}, testDir); ctx.setEnforcer(new Enforcer()); @@ -267,7 +267,7 @@ public void testGetFromDescendingOrder() throws Exception { FileFragment[] frags = sm.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); TaskAttemptContext - ctx = new TaskAttemptContext(conf, new QueryContext(conf), + ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] {frags[0]}, testDir); ctx.setEnforcer(new Enforcer()); From ea952a4a3068f9ccff8cf2455300ddd7968da0fd Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 31 Jul 2014 03:09:19 +0900 Subject: [PATCH 11/13] Fixed all unit tests. --- .../java/org/apache/tajo/SessionVars.java | 116 +++++++++----- .../java/org/apache/tajo/conf/TajoConf.java | 142 ++++++++---------- .../tajo/engine/planner/LogicalOptimizer.java | 8 +- .../tajo/engine/planner/LogicalPlan.java | 4 +- .../planner/LogicalPlanPreprocessor.java | 18 +-- .../engine/planner/LogicalPlanVerifier.java | 12 +- .../tajo/engine/planner/LogicalPlanner.java | 35 +++-- .../engine/planner/PhysicalPlannerImpl.java | 12 +- .../planner/PreLogicalPlanVerifier.java | 21 +-- .../engine/planner/global/GlobalPlanner.java | 18 +-- .../planner/physical/ExternalSortExec.java | 4 +- .../planner/physical/StoreTableExec.java | 3 +- .../tajo/engine/query/QueryContext.java | 48 ++---- .../org/apache/tajo/master/GlobalEngine.java | 92 +++++------- .../tajo/master/TajoMasterClientService.java | 33 +++- .../tajo/master/TaskSchedulerFactory.java | 1 + .../apache/tajo/master/querymaster/Query.java | 3 +- .../master/querymaster/QueryMasterTask.java | 4 +- .../master/querymaster/Repartitioner.java | 17 +-- .../tajo/master/querymaster/SubQuery.java | 8 +- .../apache/tajo/master/session/Session.java | 44 +++--- .../tajo/util/metrics/TajoSystemMetrics.java | 2 +- .../java/org/apache/tajo/worker/Task.java | 2 +- .../apache/tajo/LocalTajoTestingUtility.java | 5 + .../org/apache/tajo/QueryTestCaseBase.java | 2 +- .../apache/tajo/client/TestTajoClient.java | 7 +- .../apache/tajo/engine/eval/ExprTestBase.java | 9 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 15 +- .../engine/planner/TestLogicalOptimizer.java | 19 ++- .../tajo/engine/planner/TestLogicalPlan.java | 2 +- .../engine/planner/TestLogicalPlanner.java | 135 ++++++++++++----- .../tajo/engine/planner/TestPlannerUtil.java | 4 +- .../planner/global/TestBroadcastJoinPlan.java | 34 +++-- .../planner/physical/TestBNLJoinExec.java | 5 +- .../planner/physical/TestBSTIndexExec.java | 2 +- .../physical/TestExternalSortExec.java | 2 +- .../physical/TestFullOuterHashJoinExec.java | 15 +- .../physical/TestFullOuterMergeJoinExec.java | 17 ++- .../physical/TestHashAntiJoinExec.java | 2 +- .../planner/physical/TestHashJoinExec.java | 10 +- .../physical/TestHashSemiJoinExec.java | 2 +- .../physical/TestLeftOuterHashJoinExec.java | 17 +-- .../physical/TestLeftOuterNLJoinExec.java | 15 +- .../planner/physical/TestMergeJoinExec.java | 2 +- .../planner/physical/TestNLJoinExec.java | 13 +- .../planner/physical/TestPhysicalPlanner.java | 52 ++++--- .../TestProgressExternalSortExec.java | 2 +- .../physical/TestRightOuterHashJoinExec.java | 9 +- .../physical/TestRightOuterMergeJoinExec.java | 19 +-- .../engine/planner/physical/TestSortExec.java | 2 +- .../tajo/engine/query/TestGroupByQuery.java | 6 +- .../tajo/engine/query/TestJoinBroadcast.java | 4 +- .../tajo/engine/query/TestJoinQuery.java | 74 ++++----- .../tajo/engine/query/TestSelectQuery.java | 6 +- .../tajo/engine/query/TestSortQuery.java | 4 +- .../tajo/master/TestExecutionBlockCursor.java | 4 +- .../apache/tajo/master/TestGlobalPlanner.java | 2 +- .../TestQueryUnitStatusUpdate.java | 2 +- .../worker/TestRangeRetrieverHandler.java | 4 +- 59 files changed, 639 insertions(+), 532 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index cb489f9b34..66f740a4a6 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -19,7 +19,6 @@ package org.apache.tajo; import com.google.common.collect.Maps; -import org.apache.tajo.annotation.Nullable; import java.util.Map; @@ -28,51 +27,92 @@ public enum SessionVars implements ConfigKey { + // Common Suffix Naming Rules: + // + // * LIMIT - We use the suffix 'LIMIT' if the variable is threshold. So, if some value is greater or less than + // the variable with suffix 'LIMIT', some action will be different from before. + // * SIZE - The suffix 'SIZE' means a data volume like bytes or mega bytes. + // It should be used for user's desired volume. + // * ENABLED - The suffix 'ENABLED' means a true or false value. If it is true, it will enable some feature. + // Otherwise, the feature will be turned off. + + //------------------------------------------------------------------------------- // Server Side Only Variables //------------------------------------------------------------------------------- - USER_NAME(ConfVars.USERNAME, "user name", SERVER_SIDE_VAR), + SESSION_ID(ConfVars.$EMPTY, "session variable", SERVER_SIDE_VAR), + SESSION_LAST_ACCESS_TIME(ConfVars.$EMPTY, "last access time", SERVER_SIDE_VAR), + + USERNAME(ConfVars.USERNAME, "username", SERVER_SIDE_VAR), + CLIENT_HOST(ConfVars.$EMPTY, "client hostname", SERVER_SIDE_VAR), + CURRENT_DATABASE(ConfVars.$EMPTY, "current database", SERVER_SIDE_VAR), //------------------------------------------------------------------------------- // Client Side Variables //------------------------------------------------------------------------------- - // Client Connection - CLIENT_SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "", DEFAULT), + // Client -------------------------------------------------------- + SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "", DEFAULT), + + // Command line interface and its behavior -------------------------------- + // TODO - they should be replaced by '\pset' meta command + DISPLAY_WIDTH(ConfVars.$CLI_MAX_COLUMN, "", DEFAULT), + DISPLAY_FORMATTER_CLASS(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, "", DEFAULT), + DISPLAY_NULL_CHAR(ConfVars.$CLI_NULL_CHAR, "", DEFAULT), - // Command line interface - CLI_MAX_COLUMN(ConfVars.$CLI_MAX_COLUMN, "", DEFAULT), CLI_PRINT_PAUSE_NUM_RECORDS(ConfVars.$CLI_PRINT_PAUSE_NUM_RECORDS, "", DEFAULT), CLI_PRINT_PAUSE(ConfVars.$CLI_PRINT_PAUSE, "", DEFAULT), CLI_PRINT_ERROR_TRACE(ConfVars.$CLI_PRINT_ERROR_TRACE, "", DEFAULT), - CLI_OUTPUT_FORMATTER_CLASS(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, "", DEFAULT), - CLI_NULL_CHAR(ConfVars.$CLI_NULL_CHAR, "", DEFAULT), ON_ERROR_STOP(ConfVars.$CLI_ERROR_STOP, "tsql will exist if an error occurs.", DEFAULT), - // Timezone & Date + // Timezone & Date ---------------------------------------------------------- TZ(ConfVars.$TIMEZONE, "Timezone", FROM_SHELL_ENV), DATE_ORDER(ConfVars.$DATE_ORDER, "YMD", FROM_SHELL_ENV), - // Locales and Character set (reserved variables, which are currently not used. - LANG(null, "Language", FROM_SHELL_ENV), - LC_ALL(null, "String sort order", FROM_SHELL_ENV), - LC_COLLATE(null, "String sort order", FROM_SHELL_ENV), - LC_CTYPE(null, "Character classification (What is a letter? Its upper-case equivalent?)", FROM_SHELL_ENV), - LC_MESSAGES(null, "Language of messages", FROM_SHELL_ENV), - LC_MONETARY(null, "Formatting of currency amounts", FROM_SHELL_ENV), - LC_NUMERIC(null, "Formatting of numbers", FROM_SHELL_ENV), - LC_TIME(null, "Formatting of dates and times", FROM_SHELL_ENV), + // Locales and Character set ------------------------------------------------ + // TODO - they are reserved variables, and we should support them. + LANG(ConfVars.$EMPTY, "Language", FROM_SHELL_ENV), + LC_ALL(ConfVars.$EMPTY, "String sort order", FROM_SHELL_ENV), + LC_COLLATE(ConfVars.$EMPTY, "String sort order", FROM_SHELL_ENV), + LC_CTYPE(ConfVars.$EMPTY, "Character classification (What is a letter? Its upper-case equivalent?)", FROM_SHELL_ENV), + LC_MESSAGES(ConfVars.$EMPTY, "Language of messages", FROM_SHELL_ENV), + LC_MONETARY(ConfVars.$EMPTY, "Formatting of currency amounts", FROM_SHELL_ENV), + LC_NUMERIC(ConfVars.$EMPTY, "Formatting of numbers", FROM_SHELL_ENV), + LC_TIME(ConfVars.$EMPTY, "Formatting of dates and times", FROM_SHELL_ENV), + + + // Query and Optimization --------------------------------------------------- + + // for distributed query strategies + BROADCAST_TABLE_SIZE_LIMIT(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, "", DEFAULT), + + JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size", DEFAULT), + SORT_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_SORT_TASK_VOLUME, "sort task input size", DEFAULT), + GROUPBY_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_GROUPBY_TASK_VOLUME, "group by task input size", DEFAULT), + + JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join", DEFAULT), + GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort", DEFAULT), + TABLE_PARTITION_WRITE_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME, // TODO - rename + "shuffle output size for partition table write", DEFAULT), - // Query and Optimization - OUTPUT_PER_FILE_SIZE(null, "", DEFAULT) + // for physical Executors + EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort", DEFAULT), + HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "hash join size limit", DEFAULT), + HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD, "hash groupby size limit", DEFAULT), + MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size", DEFAULT), + NULL_CHAR(ConfVars.$CSVFILE_NULL, "null char of text files", DEFAULT), //------------------------------------------------------------------------------- // Only for Unit Testing //------------------------------------------------------------------------------- - + DEBUG_ENABLED(ConfVars.$DEBUG_ENABLED, "(debug only) debug mode enabled", DEFAULT), + TEST_BROADCAST_JOIN_ENABLED(ConfVars.$TEST_BROADCAST_JOIN_ENABLED, "(test only) broadcast enabled", DEFAULT), + TEST_JOIN_OPT_ENABLED(ConfVars.$TEST_JOIN_OPT_ENABLED, "(test only) join optimization enabled", DEFAULT), + TEST_FILTER_PUSHDOWN_ENABLED(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED, "filter push down enabled", DEFAULT), + TEST_MIN_TASK_NUM(ConfVars.$TEST_MIN_TASK_NUM, "(test only) min task num", DEFAULT), ; private static Map SESSION_VARS = Maps.newHashMap(); @@ -88,31 +128,19 @@ public enum SessionVars implements ConfigKey { private final VariableMode mode; public static enum VariableMode { - DEFAULT, - SERVER_SIDE_VAR, - FROM_SHELL_ENV - } - - public static boolean exists(String name) { - return SESSION_VARS.containsKey(name.toUpperCase()); - } - - public static SessionVars get(String keyname) { - if (exists(keyname)) { - return SESSION_VARS.get(keyname.toUpperCase()); - } else { - return null; - } + DEFAULT, // Client can set or change variables of this mode.. + FROM_SHELL_ENV, // This is similar to DEFAULT mode. In addition, it tries to get values from shell env. variables. + SERVER_SIDE_VAR, // only TajoMaster is able to set and change variables of this mode. } - SessionVars(@Nullable ConfVars key, String description, VariableMode mode) { + SessionVars(ConfVars key, String description, VariableMode mode) { this.key = key; this.description = description; this.mode = mode; } public String keyname() { - return SESSION_PREFIX + key.name(); + return SESSION_PREFIX + name(); } public ConfigType type() { @@ -134,4 +162,16 @@ public String getDescription() { public VariableMode getMode() { return mode; } + + public static boolean exists(String name) { + return SESSION_VARS.containsKey(name.toUpperCase()); + } + + public static SessionVars get(String keyname) { + if (exists(keyname)) { + return SESSION_VARS.get(keyname.toUpperCase()); + } else { + return null; + } + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 62031a0de2..a5241ba0ef 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -136,9 +136,11 @@ public static int setDateOrder(int dateOrder) { public static enum ConfVars implements ConfigKey { - ////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////////// // Tajo System Configuration - ////////////////////////////////// + // + // They are all static configs which are not changed or not overwritten at all. + /////////////////////////////////////////////////////////////////////////////////////// // a username for a running Tajo cluster ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/"), @@ -197,23 +199,19 @@ public static enum ConfVars implements ConfigKey { // Catalog CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005"), - ////////////////////////////////// - // for Yarn Resource Manager - ////////////////////////////////// + + // for Yarn Resource Manager ---------------------------------------------- + /** how many launching TaskRunners in parallel */ YARN_RM_QUERY_MASTER_MEMORY_MB("tajo.querymaster.memory-mb", 512), YARN_RM_QUERY_MASTER_DISKS("tajo.yarn-rm.querymaster.disks", 1), YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM("tajo.yarn-rm.parallel-task-runner-launcher-num", 16), YARN_RM_WORKER_NUMBER_PER_NODE("tajo.yarn-rm.max-worker-num-per-node", 8), - ////////////////////////////////// // Query Configuration - ////////////////////////////////// QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60), - ////////////////////////////////// - // Shuffle Configuration - ////////////////////////////////// + // Shuffle Configuration -------------------------------------------------- PULLSERVER_PORT("tajo.pullserver.port", 0), SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false), SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"), @@ -222,61 +220,28 @@ public static enum ConfVars implements ConfigKey { SHUFFLE_FETCHER_READ_TIMEOUT("tajo.shuffle.fetcher.read.timeout-sec", 5), SHUFFLE_FETCHER_READ_RETRY_MAX_NUM("tajo.shuffle.fetcher.read.retry.max-num", 5), - ////////////////////////////////// - // Storage Configuration - ////////////////////////////////// + + // Storage Configuration -------------------------------------------------- ROWFILE_SYNC_INTERVAL("rowfile.sync.interval", 100), MINIMUM_SPLIT_SIZE("tajo.min.split.size", (long) 1), // for RCFile HIVEUSEEXPLICITRCFILEHEADER("tajo.exec.rcfile.use.explicit.header", true), - // for Storage Manager v2 + // for Storage Manager ---------------------------------------------------- STORAGE_MANAGER_VERSION_2("tajo.storage-manager.v2", false), STORAGE_MANAGER_DISK_SCHEDULER_MAX_READ_BYTES_PER_SLOT("tajo.storage-manager.max-read-bytes", 8 * 1024 * 1024), STORAGE_MANAGER_DISK_SCHEDULER_REPORT_INTERVAL("tajo.storage-manager.disk-scheduler.report-interval", 60 * 1000), STORAGE_MANAGER_CONCURRENCY_PER_DISK("tajo.storage-manager.disk-scheduler.per-disk-concurrency", 2), - ////////////////////////////////////////// - // Distributed Query Execution Parameters - ////////////////////////////////////////// - DIST_QUERY_BROADCAST_JOIN_AUTO("tajo.dist-query.join.auto-broadcast", true), - DIST_QUERY_BROADCAST_JOIN_THRESHOLD("tajo.dist-query.join.broadcast.threshold-bytes", (long)5 * 1048576), - - DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 128), - DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128), - DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128), - - DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128), - DIST_QUERY_SORT_PARTITION_VOLUME("tajo.dist-query.sort.partition-volume-mb", 256), - DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256), - - DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256), - - ////////////////////////////////// - // Physical Executors - ////////////////////////////////// - EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1), - EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L), - EXECUTOR_EXTERNAL_SORT_FANOUT("tajo.executor.external-sort.fanout-num", 8), - - EXECUTOR_INNER_JOIN_INMEMORY_HASH_TABLE_SIZE("tajo.executor.join.inner.in-memory-table-num", (long)1000000), - EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD("tajo.executor.join.inner.in-memory-hash-threshold-bytes", - (long)256 * 1048576), - EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD("tajo.executor.join.outer.in-memory-hash-threshold-bytes", - (long)256 * 1048576), - EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-bytes", - (long)256 * 1048576), - ////////////////////////////////// - // RPC - ////////////////////////////////// + // RPC -------------------------------------------------------------------- RPC_POOL_MAX_IDLE("tajo.rpc.pool.idle.max", 10), - //Internal RPC Client + // Internal RPC Client INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM("tajo.internal.rpc.client.worker-thread-num", Runtime.getRuntime().availableProcessors() * 2), - //Internal RPC Server + // Internal RPC Server MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.master.rpc.server.worker-thread-num", Runtime.getRuntime().availableProcessors() * 2), QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.querymaster.rpc.server.worker-thread-num", @@ -297,19 +262,24 @@ public static enum ConfVars implements ConfigKey { WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM("tajo.worker.service.rpc.server.worker-thread-num", Runtime.getRuntime().availableProcessors() * 1), - ////////////////////////////////// - // The Below is reserved - ////////////////////////////////// - - // GeoIP - GEOIP_DATA("tajo.function.geoip-database-location", ""), - - ////////////////////////////////// - // Task Configuration + // Task Configuration ----------------------------------------------------- TASK_DEFAULT_MEMORY("tajo.task.memory-slot-mb.default", 512), TASK_DEFAULT_DISK("tajo.task.disk-slot.default", 0.5f), TASK_DEFAULT_SIZE("tajo.task.size-mb", 128), - ////////////////////////////////// + + // Query and Optimization ------------------------------------------------- + EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1), + EXECUTOR_EXTERNAL_SORT_FANOUT("tajo.executor.external-sort.fanout-num", 8), + + EXECUTOR_INNER_JOIN_INMEMORY_HASH_TABLE_SIZE("tajo.executor.join.inner.in-memory-table-num", (long)1000000), + + // Metrics ---------------------------------------------------------------- + METRICS_PROPERTY_FILENAME("tajo.metrics.property.file", "tajo-metrics.properties"), + + // Misc ------------------------------------------------------------------- + + // Geo IP + GEOIP_DATA("tajo.function.geoip-database-location", ""), ///////////////////////////////////////////////////////////////////////////////// // User Session Configuration @@ -317,40 +287,60 @@ public static enum ConfVars implements ConfigKey { // All session variables begin with dollor($) sign. They are default configs // for session variables. Do not directly use the following configs. Instead, // please use QueryContext in order to access session variables. + // + // Also, users can change the default values of session variables in tajo-site.xml. ///////////////////////////////////////////////////////////////////////////////// - $CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour. - // Metrics - $METRICS_PROPERTY_FILENAME("tajo.metrics.property.file", "tajo-metrics.properties"), + $EMPTY("tajo._", ""), + + // Query and Optimization --------------------------------------------------- + + // for distributed query strategies + $DIST_QUERY_BROADCAST_JOIN_THRESHOLD("tajo.dist-query.join.broadcast.threshold-bytes", (long)5 * 1048576), + + $DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 128), + $DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128), + $DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128), + + $DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128), + $DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256), + $DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256), - //CLI + // for physical Executors + $EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L), + $EXECUTOR_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.common.in-memory-hash-threshold-bytes", + (long)256 * 1048576), + $EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-bytes", + (long)256 * 1048576), + $MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite + + + // Client ----------------------------------------------------------------- + $CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour. + + // Command line interface and its behavior -------------------------------- $CLI_MAX_COLUMN("tajo.cli.max_columns", 120), + $CLI_NULL_CHAR("tajo.cli.nullchar", ""), $CLI_PRINT_PAUSE_NUM_RECORDS("tajo.cli.print.pause.num.records", 100), $CLI_PRINT_PAUSE("tajo.cli.print.pause", true), $CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true), $CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"), - $CLI_NULL_CHAR("tajo.cli.nullchar", ""), $CLI_ERROR_STOP("tajo.cli.error.stop", false), - // TIME & DATE + // Timezone & Date ---------------------------------------------------------- $TIMEZONE("tajo.timezone", System.getProperty("user.timezone")), $DATE_ORDER("tajo.date.order", "YMD"), - //PLANNER - $PLANNER_USE_FILTER_PUSHDOWN("tajo.planner.use.filter.pushdown", true), - // FILE FORMAT $CSVFILE_NULL("tajo.csvfile.null", "\\\\N"), - //OPTIMIZER - $OPTIMIZER_JOIN_ENABLE("tajo.optimizer.join.enable", true), - - // DEBUG OPTION - $TAJO_DEBUG("tajo.debug", false), - - // ONLY FOR TESTCASE - $TESTCASE_MIN_TASK_NUM("tajo.testcase.min.task.num", -1) + // Only for Debug and Testing + $DEBUG_ENABLED("tajo.debug.enabled", false), + $TEST_BROADCAST_JOIN_ENABLED("tajo.dist-query.join.auto-broadcast", true), + $TEST_JOIN_OPT_ENABLED("tajo.test.plan.join-optimization.enabled", true), + $TEST_FILTER_PUSHDOWN_ENABLED("tajo.test.plan.filter-pushdown.enabled", true), + $TEST_MIN_TASK_NUM("tajo.test.min-task-num", -1) ; public final String varname; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java index f9a911c37a..fd10bc9db9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; @@ -35,6 +36,7 @@ import org.apache.tajo.engine.planner.logical.join.JoinGraph; import org.apache.tajo.engine.planner.logical.join.JoinOrderAlgorithm; import org.apache.tajo.engine.planner.rewrite.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import java.util.LinkedHashSet; @@ -57,7 +59,7 @@ public class LogicalOptimizer { public LogicalOptimizer(TajoConf systemConf) { rulesBeforeJoinOpt = new BasicQueryRewriteEngine(); - if (systemConf.getBoolVar(ConfVars.$PLANNER_USE_FILTER_PUSHDOWN)) { + if (systemConf.getBoolVar(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED)) { rulesBeforeJoinOpt.addRewriteRule(new FilterPushDownRule()); } @@ -84,13 +86,13 @@ public LogicalNode optimize(LogicalPlan plan) throws PlanningException { return optimize(null, plan); } - public LogicalNode optimize(Session session, LogicalPlan plan) throws PlanningException { + public LogicalNode optimize(QueryContext context, LogicalPlan plan) throws PlanningException { rulesBeforeJoinOpt.rewrite(plan); DirectedGraphCursor blockCursor = new DirectedGraphCursor(plan.getQueryBlockGraph(), plan.getRootBlock().getName()); - if (session == null || "true".equals(session.getVariable(ConfVars.$OPTIMIZER_JOIN_ENABLE.varname, "true"))) { + if (context == null || context.getBool(SessionVars.TEST_JOIN_OPT_ENABLED)) { // default is true while (blockCursor.hasNext()) { optimizeJoinOrder(plan, blockCursor.nextBlock()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java index 86bacefbd5..ee65b2b62f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java @@ -67,10 +67,8 @@ public class LogicalPlan { LogicalPlanner planner; private boolean isExplain; - private final String currentDatabase; - public LogicalPlan(String currentDatabase, LogicalPlanner planner) { - this.currentDatabase = currentDatabase; + public LogicalPlan(LogicalPlanner planner) { this.planner = planner; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index 84fe6c2fb9..6ee0ff8bef 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -26,10 +26,10 @@ import org.apache.tajo.engine.exception.NoSuchColumnException; import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.engine.planner.nameresolver.NameResolver; +import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.utils.SchemaUtil; -import org.apache.tajo.master.session.Session; import org.apache.tajo.util.TUtil; import java.util.*; @@ -42,18 +42,18 @@ public class LogicalPlanPreprocessor extends BaseAlgebraVisitor stack, Relat if (CatalogUtil.isFQTableName(expr.getName())) { actualRelationName = relation.getName(); } else { - actualRelationName = CatalogUtil.buildFQName(ctx.session.getCurrentDatabase(), relation.getName()); + actualRelationName = CatalogUtil.buildFQName(ctx.queryContext.getCurrentDatabase(), relation.getName()); } TableDesc desc = catalog.getTableDesc(actualRelationName); @@ -388,7 +388,7 @@ public LogicalNode visitTableSubQuery(PreprocessContext ctx, Stack stack, // a table subquery should be dealt as a relation. TableSubQueryNode node = ctx.plan.createNode(TableSubQueryNode.class); - node.init(CatalogUtil.buildFQName(ctx.session.getCurrentDatabase(), expr.getName()), child); + node.init(CatalogUtil.buildFQName(ctx.queryContext.getCurrentDatabase(), expr.getName()), child); ctx.currentBlock.addRelation(node); return node; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java index bb8192fd9a..6512ae0233 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java @@ -24,7 +24,7 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.master.session.Session; +import org.apache.tajo.engine.query.QueryContext; import java.util.Stack; @@ -38,17 +38,17 @@ public LogicalPlanVerifier(TajoConf conf, CatalogService catalog) { } public static class Context { - Session session; + QueryContext queryContext; VerificationState state; - public Context(Session session, VerificationState state) { - this.session = session; + public Context(QueryContext queryContext, VerificationState state) { + this.queryContext = this.queryContext; this.state = state; } } - public VerificationState verify(Session session, VerificationState state, LogicalPlan plan) throws PlanningException { - Context context = new Context(session, state); + public VerificationState verify(QueryContext queryContext, VerificationState state, LogicalPlan plan) throws PlanningException { + Context context = new Context(queryContext, state); visit(context, plan, plan.getRootBlock()); return context.state; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index a4820cb46e..35df11fe64 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -42,17 +42,16 @@ import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.utils.SchemaUtil; -import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.StorageUtil; -import org.apache.tajo.util.Pair; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import java.util.*; import static org.apache.tajo.algebra.CreateTable.PartitionType; - import static org.apache.tajo.engine.planner.ExprNormalizer.ExprNormalizedResult; import static org.apache.tajo.engine.planner.LogicalPlan.BlockType; import static org.apache.tajo.engine.planner.LogicalPlanPreprocessor.PreprocessContext; @@ -75,7 +74,7 @@ public LogicalPlanner(CatalogService catalog) { } public static class PlanContext { - Session session; + QueryContext queryContext; LogicalPlan plan; // transient data for each query block @@ -83,15 +82,15 @@ public static class PlanContext { boolean debugOrUnitTests; - public PlanContext(Session session, LogicalPlan plan, QueryBlock block, boolean debugOrUnitTests) { - this.session = session; + public PlanContext(QueryContext context, LogicalPlan plan, QueryBlock block, boolean debugOrUnitTests) { + this.queryContext = context; this.plan = plan; this.queryBlock = block; this.debugOrUnitTests = debugOrUnitTests; } public PlanContext(PlanContext context, QueryBlock block) { - this.session = context.session; + this.queryContext = context.queryContext; this.plan = context.plan; this.queryBlock = block; this.debugOrUnitTests = context.debugOrUnitTests; @@ -109,21 +108,21 @@ public String toString() { * @param expr A relational algebraic expression for a query. * @return A logical plan */ - public LogicalPlan createPlan(Session session, Expr expr) throws PlanningException { - return createPlan(session, expr, false); + public LogicalPlan createPlan(QueryContext context, Expr expr) throws PlanningException { + return createPlan(context, expr, false); } @VisibleForTesting - public LogicalPlan createPlan(Session session, Expr expr, boolean debug) throws PlanningException { + public LogicalPlan createPlan(QueryContext queryContext, Expr expr, boolean debug) throws PlanningException { - LogicalPlan plan = new LogicalPlan(session.getCurrentDatabase(), this); + LogicalPlan plan = new LogicalPlan(this); QueryBlock rootBlock = plan.newAndGetBlock(LogicalPlan.ROOT_BLOCK); - PreprocessContext preProcessorCtx = new PreprocessContext(session, plan, rootBlock); + PreprocessContext preProcessorCtx = new PreprocessContext(queryContext, plan, rootBlock); preprocessor.visit(preProcessorCtx, new Stack(), expr); plan.resetGeneratedId(); - PlanContext context = new PlanContext(session, plan, plan.getRootBlock(), debug); + PlanContext context = new PlanContext(queryContext, plan, plan.getRootBlock(), debug); LogicalNode topMostNode = this.visit(context, new Stack(), expr); // Add Root Node @@ -1424,7 +1423,7 @@ private InsertNode buildInsertIntoTablePlan(PlanContext context, InsertNode inse databaseName = CatalogUtil.extractQualifier(expr.getTableName()); tableName = CatalogUtil.extractSimpleName(expr.getTableName()); } else { - databaseName = context.session.getCurrentDatabase(); + databaseName = context.queryContext.getCurrentDatabase(); tableName = expr.getTableName(); } TableDesc desc = catalog.getTableDesc(databaseName, tableName); @@ -1624,7 +1623,7 @@ public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, if (CatalogUtil.isFQTableName(parentTableName) == false) { parentTableName = - CatalogUtil.buildFQName(context.session.getCurrentDatabase(), + CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), parentTableName); } TableDesc parentTableDesc = catalog.getTableDesc(parentTableName); @@ -1657,7 +1656,7 @@ public LogicalNode visitCreateTable(PlanContext context, Stack stack, Crea createTableNode.setTableName(expr.getTableName()); } else { createTableNode.setTableName( - CatalogUtil.buildFQName(context.session.getCurrentDatabase(), expr.getTableName())); + CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), expr.getTableName())); } // This is CREATE TABLE LIKE if(expr.getLikeParentTableName() != null) @@ -1753,7 +1752,7 @@ private PartitionMethodDesc getPartitionMethod(PlanContext context, CreateTable.ColumnPartition partition = (CreateTable.ColumnPartition) expr; String partitionExpression = Joiner.on(',').join(partition.getColumns()); - partitionMethodDesc = new PartitionMethodDesc(context.session.getCurrentDatabase(), tableName, + partitionMethodDesc = new PartitionMethodDesc(context.queryContext.getCurrentDatabase(), tableName, CatalogProtos.PartitionType.COLUMN, partitionExpression, convertColumnsToSchema(partition.getColumns())); } else { throw new PlanningException(String.format("Not supported PartitonType: %s", expr.getPartitionType())); @@ -1816,7 +1815,7 @@ public LogicalNode visitDropTable(PlanContext context, Stack stack, DropTa if (CatalogUtil.isFQTableName(dropTable.getTableName())) { qualified = dropTable.getTableName(); } else { - qualified = CatalogUtil.buildFQName(context.session.getCurrentDatabase(), dropTable.getTableName()); + qualified = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), dropTable.getTableName()); } dropTableNode.init(qualified, dropTable.isIfExists(), dropTable.isPurge()); return dropTableNode; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index c5d018bca3..d065d18556 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -37,7 +38,6 @@ import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.physical.*; -import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InternalException; import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer; @@ -58,7 +58,6 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType; -import static org.apache.tajo.conf.TajoConf.ConfVars; import static org.apache.tajo.ipc.TajoWorkerProtocol.ColumnPartitionEnforcer.ColumnPartitionAlgorithm; import static org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty; import static org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty.EnforceType; @@ -256,8 +255,7 @@ public boolean checkIfInMemoryInnerJoinIsPossible(TaskAttemptContext context, Lo throws IOException { String [] lineage = PlannerUtil.getRelationLineage(node); long volume = estimateSizeRecursive(context, lineage); - boolean inMemoryInnerJoinFlag = volume <= - context.getQueryContext().getLong(ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD); + boolean inMemoryInnerJoinFlag = volume <= context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT); LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.", context.getTaskId().toString(), (left ? "Left" : "Right"), @@ -470,7 +468,7 @@ private PhysicalExec createBestLeftOuterJoinPlan(TaskAttemptContext context, Joi String [] rightLineage = PlannerUtil.getRelationLineage(plan.getRightChild()); long rightTableVolume = estimateSizeRecursive(context, rightLineage); - if (rightTableVolume < context.getQueryContext().getLong(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)) { + if (rightTableVolume < context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT)) { // we can implement left outer join using hash join, using the right operand as the build relation LOG.info("Left Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, leftExec, rightExec); @@ -488,7 +486,7 @@ private PhysicalExec createBestRightJoinPlan(TaskAttemptContext context, JoinNod // blocking, but merge join is blocking as well) String [] outerLineage4 = PlannerUtil.getRelationLineage(plan.getLeftChild()); long outerSize = estimateSizeRecursive(context, outerLineage4); - if (outerSize < context.getQueryContext().getLong(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD)){ + if (outerSize < context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT)){ LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, rightExec, leftExec); } else { @@ -970,7 +968,7 @@ private PhysicalExec createBestAggregationPlan(TaskAttemptContext context, Group String [] outerLineage = PlannerUtil.getRelationLineage(groupbyNode.getChild()); long estimatedSize = estimateSizeRecursive(context, outerLineage); - final long threshold = context.getQueryContext().getLong(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD); + final long threshold = context.getQueryContext().getLong(SessionVars.HASH_GROUPBY_SIZE_LIMIT); // if the relation size is less than the threshold, // the hash aggregation will be used. diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java index 2d6c095a18..75dcc18180 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java @@ -24,7 +24,7 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.master.session.Session; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.util.TUtil; import java.util.Set; @@ -38,17 +38,17 @@ public PreLogicalPlanVerifier(CatalogService catalog) { } public static class Context { - Session session; + QueryContext queryContext; VerificationState state; - public Context(Session session, VerificationState state) { - this.session = session; + public Context(QueryContext queryContext, VerificationState state) { + this.queryContext = queryContext; this.state = state; } } - public VerificationState verify(Session session, VerificationState state, Expr expr) throws PlanningException { - Context context = new Context(session, state); + public VerificationState verify(QueryContext queryContext, VerificationState state, Expr expr) throws PlanningException { + Context context = new Context(queryContext, state); visit(context, new Stack(), expr); return context.state; } @@ -127,7 +127,7 @@ private boolean assertRelationExistence(Context context, String tableName) { if (CatalogUtil.isFQTableName(tableName)) { qualifiedName = tableName; } else { - qualifiedName = CatalogUtil.buildFQName(context.session.getCurrentDatabase(), tableName); + qualifiedName = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), tableName); } if (!catalog.existsTable(qualifiedName)) { @@ -143,7 +143,10 @@ private boolean assertRelationNoExistence(Context context, String tableName) { if (CatalogUtil.isFQTableName(tableName)) { qualifiedName = tableName; } else { - qualifiedName = CatalogUtil.buildFQName(context.session.getCurrentDatabase(), tableName); + qualifiedName = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), tableName); + } + if(qualifiedName == null) { + System.out.println("A"); } if (catalog.existsTable(qualifiedName)) { context.state.addVerification(String.format("relation \"%s\" already exists", qualifiedName)); @@ -246,7 +249,7 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws if (expr.hasTableName()) { String qualifiedName = expr.getTableName(); if (TajoConstants.EMPTY_STRING.equals(CatalogUtil.extractQualifier(expr.getTableName()))) { - qualifiedName = CatalogUtil.buildFQName(context.session.getCurrentDatabase(), + qualifiedName = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), expr.getTableName()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 8c0df5dbe9..432589b3a5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -38,7 +39,6 @@ import org.apache.tajo.engine.planner.global.builder.DistinctGroupbyBuilder; import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule; -import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InternalException; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; @@ -121,8 +121,8 @@ public void build(MasterPlan masterPlan) throws IOException, PlanningException { LogicalNode inputPlan = PlannerUtil.clone(masterPlan.getLogicalPlan(), masterPlan.getLogicalPlan().getRootBlock().getRoot()); - boolean autoBroadcast = masterPlan.getContext().getBool(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); - if (autoBroadcast) { + boolean broadcastEnabled = masterPlan.getContext().getBool(SessionVars.TEST_BROADCAST_JOIN_ENABLED); + if (broadcastEnabled) { // pre-visit the master plan in order to find tables to be broadcasted // this visiting does not make any execution block and change plan. BroadcastJoinMarkCandidateVisitor markCandidateVisitor = new BroadcastJoinMarkCandidateVisitor(); @@ -268,11 +268,11 @@ private ExecutionBlock buildJoinPlan(GlobalPlanContext context, JoinNode joinNod MasterPlan masterPlan = context.plan; ExecutionBlock currentBlock; - boolean autoBroadcast = context.getPlan().getContext().getBool(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO); - long broadcastThreshold = context.getPlan().getContext().getLong(ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD); + boolean broadcastEnabled = context.getPlan().getContext().getBool(SessionVars.TEST_BROADCAST_JOIN_ENABLED); + long broadcastTableSizeLimit = context.getPlan().getContext().getLong(SessionVars.BROADCAST_TABLE_SIZE_LIMIT); // to check when the tajo.dist-query.join.broadcast.auto property is true - if (autoBroadcast && joinNode.isCandidateBroadcast()) { + if (broadcastEnabled && joinNode.isCandidateBroadcast()) { LogicalNode leftNode = joinNode.getLeftChild(); LogicalNode rightNode = joinNode.getRightChild(); @@ -293,7 +293,7 @@ private ExecutionBlock buildJoinPlan(GlobalPlanContext context, JoinNode joinNod // Checking Left Side of Join if (ScanNode.isScanNode(leftNode)) { ScanNode scanNode = (ScanNode)leftNode; - if (joinNode.getJoinType() == JoinType.LEFT_OUTER || getTableVolume(scanNode) >= broadcastThreshold) { + if (joinNode.getJoinType() == JoinType.LEFT_OUTER || getTableVolume(scanNode) >= broadcastTableSizeLimit) { numLargeTables++; } else { leftBroadcast = true; @@ -306,7 +306,7 @@ private ExecutionBlock buildJoinPlan(GlobalPlanContext context, JoinNode joinNod // Checking Right Side OF Join if (ScanNode.isScanNode(rightNode)) { ScanNode scanNode = (ScanNode)rightNode; - if (joinNode.getJoinType() == JoinType.RIGHT_OUTER || getTableVolume(scanNode) >= broadcastThreshold) { + if (joinNode.getJoinType() == JoinType.RIGHT_OUTER || getTableVolume(scanNode) >= broadcastTableSizeLimit) { numLargeTables++; } else { rightBroadcast = true; @@ -331,7 +331,7 @@ private ExecutionBlock buildJoinPlan(GlobalPlanContext context, JoinNode joinNod } JoinNode broadcastJoinNode = (JoinNode)eachNode; ScanNode scanNode = broadcastJoinNode.getRightChild(); - if (getTableVolume(scanNode) < broadcastThreshold) { + if (getTableVolume(scanNode) < broadcastTableSizeLimit) { broadcastTargetScanNodes.add(scanNode); blockJoinNode = broadcastJoinNode; LOG.info("The table " + scanNode.getCanonicalName() + " (" diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java index f71475867c..31cb3b68ce 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.io.IOUtils; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; @@ -38,6 +39,7 @@ import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; +import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TaskAttemptContext; @@ -111,7 +113,7 @@ private ExternalSortExec(final TaskAttemptContext context, final AbstractStorage throw new PhysicalPlanningException(ConfVars.EXECUTOR_EXTERNAL_SORT_FANOUT.varname + " cannot be lower than 2"); } // TODO - sort buffer and core num should be changed to use the allocated container resource. - this.sortBufferBytesNum = context.getConf().getLongVar(ConfVars.EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE) * 1048576L; + this.sortBufferBytesNum = context.getQueryContext().getLong(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB; this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM); this.executorService = Executors.newFixedThreadPool(this.allocatedCoreNum); this.inMemoryTable = new ArrayList(100000); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index c562d4c7a9..c813710acd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner.physical; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.conf.TajoConf.ConfVars; @@ -59,7 +60,7 @@ public void init() throws IOException { appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, createTableNode.getTableSchema(), context.getOutputPath()); } else { - String nullChar = context.getQueryContext().get(ConfVars.$CSVFILE_NULL.varname, ConfVars.$CSVFILE_NULL.defaultVal); + String nullChar = context.getQueryContext().get(SessionVars.NULL_CHAR); meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, context.getOutputPath()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index b3d30bf5e1..52d5df08d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -26,6 +26,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.master.session.Session; import org.apache.tajo.util.KeyValueSet; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; @@ -84,6 +85,11 @@ public QueryContext(final TajoConf conf) { this.conf = conf; } + public QueryContext(final TajoConf conf, Session session) { + this.conf = conf; + putAll(session.getAllVariables()); + } + public QueryContext(final TajoConf conf, KeyValueSetProto proto) { super(proto); this.conf = conf; @@ -202,12 +208,16 @@ public String get(ConfigKey key) { // Query Config Specified Section //----------------------------------------------------------------------------------------------- + public String getCurrentDatabase() { + return get(SessionVars.CURRENT_DATABASE); + } + public void setUser(String username) { - put(SessionVars.USER_NAME.getConfVars(), username); + put(SessionVars.USERNAME, username); } public String getUser() { - return get(SessionVars.USER_NAME); + return get(SessionVars.USERNAME); } public void setStagingDir(Path path) { @@ -219,15 +229,6 @@ public Path getStagingDir() { return strVal != null ? new Path(strVal) : null; } - /** - * The fact that QueryContext has an output table means this query has a target table. - * In other words, this query is 'CREATE TABLE' or 'INSERT (OVERWRITE) INTO ' statement. - * This config is not set if a query has INSERT (OVERWRITE) INTO LOCATION '/path/..'. - */ - public boolean hasOutputTable() { - return get(QueryVars.OUTPUT_TABLE_NAME) != null; - } - /** * Set a target table name * @@ -237,11 +238,6 @@ public void setOutputTable(String tableName) { put(QueryVars.OUTPUT_TABLE_NAME, tableName); } - public String getOutputTable() { - String strVal = get(QueryVars.OUTPUT_TABLE_NAME); - return strVal != null ? strVal : null; - } - /** * The fact that QueryContext has an output path means this query will write the output to a specific directory. * In other words, this query is 'CREATE TABLE' or 'INSERT (OVERWRITE) INTO (
|LOCATION)' statement. @@ -297,7 +293,7 @@ public boolean equalKey(ConfigKey key, String another) { } } - public boolean assertCommandType(NodeType commandType) { + public boolean isCommandType(NodeType commandType) { return equalKey(QueryVars.COMMAND_TYPE, commandType.name()); } @@ -315,7 +311,7 @@ public void setCreateTable() { } public boolean isCreateTable() { - return assertCommandType(NodeType.CREATE_TABLE); + return isCommandType(NodeType.CREATE_TABLE); } public void setInsert() { @@ -323,20 +319,6 @@ public void setInsert() { } public boolean isInsert() { - return assertCommandType(NodeType.INSERT); - } - - public static Long getLongVar(QueryContext context, TajoConf conf, TajoConf.ConfVars key) { - if (context.get(key.varname) != null) { - String val = context.get(key.varname); - try { - return Long.valueOf(val); - } catch (NumberFormatException nfe) { - LOG.warn(nfe.getMessage()); - return conf.getLongVar(key); - } - } else { - return conf.getLongVar(key); - } + return isCommandType(NodeType.INSERT); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index a5345de490..37a56ba3ac 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -29,6 +29,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.tajo.QueryId; import org.apache.tajo.QueryIdFactory; +import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.algebra.AlterTablespaceSetType; import org.apache.tajo.algebra.Expr; @@ -116,40 +117,25 @@ public void stop() { public SubmitQueryResponse executeQuery(Session session, String query, boolean isJson) { LOG.info("Query: " + query); - QueryContext queryContext = new QueryContext(context.getConf()); - queryContext.putAll(session.getAllVariables()); + QueryContext queryContext = new QueryContext(context.getConf(), session); Expr planningContext; try { if (isJson) { planningContext = buildExpressionFromJson(query); } else { - // setting environment variables - String [] cmds = query.split(" "); - if(cmds != null) { - if(cmds[0].equalsIgnoreCase("set")) { - String[] params = cmds[1].split("="); - context.getConf().set(params[0], params[1]); - SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder(); - responseBuilder.setUserName(context.getConf().getVar(TajoConf.ConfVars.USERNAME)); - responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - responseBuilder.setResultCode(ClientProtos.ResultCode.OK); - return responseBuilder.build(); - } - } - planningContext = buildExpressionFromSql(queryContext, query); } String jsonExpr = planningContext.toJson(); - LogicalPlan plan = createLogicalPlan(session, planningContext); + LogicalPlan plan = createLogicalPlan(queryContext, planningContext); SubmitQueryResponse response = executeQueryInternal(queryContext, session, plan, query, jsonExpr); return response; } catch (Throwable t) { context.getSystemMetrics().counter("Query", "errorQuery").inc(); LOG.error("\nStack Trace:\n" + StringUtils.stringifyException(t)); SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder(); - responseBuilder.setUserName(context.getConf().getVar(TajoConf.ConfVars.USERNAME)); + responseBuilder.setUserName(queryContext.get(SessionVars.USERNAME)); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); responseBuilder.setIsForwarded(true); responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR); @@ -183,11 +169,11 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder(); responseBuilder.setIsForwarded(false); - responseBuilder.setUserName(context.getConf().getVar(TajoConf.ConfVars.USERNAME)); + responseBuilder.setUserName(queryContext.get(SessionVars.USERNAME)); if (PlannerUtil.checkIfDDLPlan(rootNode)) { context.getSystemMetrics().counter("Query", "numDDLQuery").inc(); - updateQuery(session, rootNode.getChild()); + updateQuery(queryContext, rootNode.getChild()); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); responseBuilder.setResultCode(ClientProtos.ResultCode.OK); @@ -390,7 +376,7 @@ private void insertNonFromQuery(QueryContext queryContext, InsertNode insertNode } - public QueryId updateQuery(Session session, String sql, boolean isJson) throws IOException, SQLException, PlanningException { + public QueryId updateQuery(QueryContext queryContext, String sql, boolean isJson) throws IOException, SQLException, PlanningException { try { LOG.info("SQL: " + sql); @@ -402,13 +388,13 @@ public QueryId updateQuery(Session session, String sql, boolean isJson) throws I expr = analyzer.parse(sql); } - LogicalPlan plan = createLogicalPlan(session, expr); + LogicalPlan plan = createLogicalPlan(queryContext, expr); LogicalRootNode rootNode = plan.getRootBlock().getRoot(); if (!PlannerUtil.checkIfDDLPlan(rootNode)) { throw new SQLException("This is not update query:\n" + sql); } else { - updateQuery(session, rootNode.getChild()); + updateQuery(queryContext, rootNode.getChild()); return QueryIdFactory.NULL_QUERY_ID; } } catch (Exception e) { @@ -417,46 +403,46 @@ public QueryId updateQuery(Session session, String sql, boolean isJson) throws I } } - private boolean updateQuery(Session session, LogicalNode root) throws IOException { + private boolean updateQuery(QueryContext queryContext, LogicalNode root) throws IOException { switch (root.getType()) { case CREATE_DATABASE: CreateDatabaseNode createDatabase = (CreateDatabaseNode) root; - createDatabase(session, createDatabase.getDatabaseName(), null, createDatabase.isIfNotExists()); + createDatabase(queryContext, createDatabase.getDatabaseName(), null, createDatabase.isIfNotExists()); return true; case DROP_DATABASE: DropDatabaseNode dropDatabaseNode = (DropDatabaseNode) root; - dropDatabase(session, dropDatabaseNode.getDatabaseName(), dropDatabaseNode.isIfExists()); + dropDatabase(queryContext, dropDatabaseNode.getDatabaseName(), dropDatabaseNode.isIfExists()); return true; case CREATE_TABLE: CreateTableNode createTable = (CreateTableNode) root; - createTable(session, createTable, createTable.isIfNotExists()); + createTable(queryContext, createTable, createTable.isIfNotExists()); return true; case DROP_TABLE: DropTableNode dropTable = (DropTableNode) root; - dropTable(session, dropTable.getTableName(), dropTable.isIfExists(), dropTable.isPurge()); + dropTable(queryContext, dropTable.getTableName(), dropTable.isIfExists(), dropTable.isPurge()); return true; case ALTER_TABLESPACE: AlterTablespaceNode alterTablespace = (AlterTablespaceNode) root; - alterTablespace(session, alterTablespace); + alterTablespace(queryContext, alterTablespace); return true; case ALTER_TABLE: AlterTableNode alterTable = (AlterTableNode) root; - alterTable(session,alterTable); + alterTable(queryContext,alterTable); return true; case TRUNCATE_TABLE: TruncateTableNode truncateTable = (TruncateTableNode) root; - truncateTable(session, truncateTable); + truncateTable(queryContext, truncateTable); return true; default: throw new InternalError("updateQuery cannot handle such query: \n" + root.toJson()); } } - private LogicalPlan createLogicalPlan(Session session, Expr expression) throws PlanningException { + private LogicalPlan createLogicalPlan(QueryContext queryContext, Expr expression) throws PlanningException { VerificationState state = new VerificationState(); - preVerifier.verify(session, state, expression); + preVerifier.verify(queryContext, state, expression); if (!state.verified()) { StringBuilder sb = new StringBuilder(); for (String error : state.getErrorMessages()) { @@ -465,19 +451,19 @@ private LogicalPlan createLogicalPlan(Session session, Expr expression) throws P throw new VerifyException(sb.toString()); } - LogicalPlan plan = planner.createPlan(session, expression); + LogicalPlan plan = planner.createPlan(queryContext, expression); if (LOG.isDebugEnabled()) { LOG.debug("============================================="); LOG.debug("Non Optimized Query: \n" + plan.toString()); LOG.debug("============================================="); } LOG.info("Non Optimized Query: \n" + plan.toString()); - optimizer.optimize(session, plan); + optimizer.optimize(queryContext, plan); LOG.info("============================================="); LOG.info("Optimized Query: \n" + plan.toString()); LOG.info("============================================="); - annotatedPlanVerifier.verify(session, state, plan); + annotatedPlanVerifier.verify(queryContext, state, plan); if (!state.verified()) { StringBuilder sb = new StringBuilder(); @@ -493,7 +479,7 @@ private LogicalPlan createLogicalPlan(Session session, Expr expression) throws P /** * Alter a given table */ - public void alterTablespace(final Session session, final AlterTablespaceNode alterTablespace) { + public void alterTablespace(final QueryContext queryContext, final AlterTablespaceNode alterTablespace) { final CatalogService catalog = context.getCatalog(); final String spaceName = alterTablespace.getTablespaceName(); @@ -517,7 +503,7 @@ public void alterTablespace(final Session session, final AlterTablespaceNode alt /** * Alter a given table */ - public void alterTable(final Session session, final AlterTableNode alterTable) throws IOException { + public void alterTable(final QueryContext queryContext, final AlterTableNode alterTable) throws IOException { final CatalogService catalog = context.getCatalog(); final String tableName = alterTable.getTableName(); @@ -529,7 +515,7 @@ public void alterTable(final Session session, final AlterTableNode alterTable) t databaseName = split[0]; simpleTableName = split[1]; } else { - databaseName = session.getCurrentDatabase(); + databaseName = queryContext.getCurrentDatabase(); simpleTableName = tableName; } final String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName); @@ -572,7 +558,8 @@ public void alterTable(final Session session, final AlterTableNode alterTable) t if (existColumnName(qualifiedName, alterTable.getNewColumnName())) { throw new ColumnNameAlreadyExistException(alterTable.getNewColumnName()); } - catalog.alterTable(CatalogUtil.renameColumn(qualifiedName, alterTable.getColumnName(), alterTable.getNewColumnName(), AlterTableType.RENAME_COLUMN)); + catalog.alterTable(CatalogUtil.renameColumn(qualifiedName, alterTable.getColumnName(), + alterTable.getNewColumnName(), AlterTableType.RENAME_COLUMN)); break; case ADD_COLUMN: if (existColumnName(qualifiedName, alterTable.getAddNewColumn().getSimpleName())) { @@ -588,7 +575,8 @@ public void alterTable(final Session session, final AlterTableNode alterTable) t /** * Truncate table a given table */ - public void truncateTable(final Session session, final TruncateTableNode truncateTableNode) throws IOException { + public void truncateTable(final QueryContext queryContext, final TruncateTableNode truncateTableNode) + throws IOException { List tableNames = truncateTableNode.getTableNames(); final CatalogService catalog = context.getCatalog(); @@ -602,7 +590,7 @@ public void truncateTable(final Session session, final TruncateTableNode truncat databaseName = split[0]; simpleTableName = split[1]; } else { - databaseName = session.getCurrentDatabase(); + databaseName = queryContext.getCurrentDatabase(); simpleTableName = eachTableName; } final String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName); @@ -641,7 +629,7 @@ private boolean existColumnName(String tableName, String columnName) { return tableDesc.getSchema().containsByName(columnName) ? true : false; } - private TableDesc createTable(Session session, CreateTableNode createTable, boolean ifNotExists) throws IOException { + private TableDesc createTable(QueryContext queryContext, CreateTableNode createTable, boolean ifNotExists) throws IOException { TableMeta meta; if (createTable.hasOptions()) { @@ -659,7 +647,7 @@ private TableDesc createTable(Session session, CreateTableNode createTable, bool databaseName = CatalogUtil.extractQualifier(createTable.getTableName()); tableName = CatalogUtil.extractSimpleName(createTable.getTableName()); } else { - databaseName = session.getCurrentDatabase(); + databaseName = queryContext.getCurrentDatabase(); tableName = createTable.getTableName(); } @@ -668,11 +656,11 @@ private TableDesc createTable(Session session, CreateTableNode createTable, bool createTable.setPath(tablePath); } - return createTableOnPath(session, createTable.getTableName(), createTable.getTableSchema(), + return createTableOnPath(queryContext, createTable.getTableName(), createTable.getTableSchema(), meta, createTable.getPath(), createTable.isExternal(), createTable.getPartitionMethod(), ifNotExists); } - public TableDesc createTableOnPath(Session session, String tableName, Schema schema, TableMeta meta, + public TableDesc createTableOnPath(QueryContext queryContext, String tableName, Schema schema, TableMeta meta, Path path, boolean isExternal, PartitionMethodDesc partitionDesc, boolean ifNotExists) throws IOException { @@ -683,7 +671,7 @@ public TableDesc createTableOnPath(Session session, String tableName, Schema sch databaseName = splitted[0]; simpleTableName = splitted[1]; } else { - databaseName = session.getCurrentDatabase(); + databaseName = queryContext.getCurrentDatabase(); simpleTableName = tableName; } String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName); @@ -736,7 +724,7 @@ public TableDesc createTableOnPath(Session session, String tableName, Schema sch } } - public boolean createDatabase(@Nullable Session session, String databaseName, + public boolean createDatabase(@Nullable QueryContext queryContext, String databaseName, @Nullable String tablespace, boolean ifNotExists) throws IOException { @@ -768,7 +756,7 @@ public boolean createDatabase(@Nullable Session session, String databaseName, return true; } - public boolean dropDatabase(Session session, String databaseName, boolean ifExists) { + public boolean dropDatabase(QueryContext queryContext, String databaseName, boolean ifExists) { boolean exists = catalog.existDatabase(databaseName); if(!exists) { @@ -780,7 +768,7 @@ public boolean dropDatabase(Session session, String databaseName, boolean ifExis } } - if (session.getCurrentDatabase().equals(databaseName)) { + if (queryContext.getCurrentDatabase().equals(databaseName)) { throw new RuntimeException("ERROR: Cannot drop the current open database"); } @@ -795,7 +783,7 @@ public boolean dropDatabase(Session session, String databaseName, boolean ifExis * @param tableName to be dropped * @param purge Remove all data if purge is true. */ - public boolean dropTable(Session session, String tableName, boolean ifExists, boolean purge) { + public boolean dropTable(QueryContext queryContext, String tableName, boolean ifExists, boolean purge) { CatalogService catalog = context.getCatalog(); String databaseName; @@ -805,7 +793,7 @@ public boolean dropTable(Session session, String tableName, boolean ifExists, bo databaseName = splitted[0]; simpleTableName = splitted[1]; } else { - databaseName = session.getCurrentDatabase(); + databaseName = queryContext.getCurrentDatabase(); simpleTableName = tableName; } String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 97f59ef734..93efe1a79a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -27,13 +27,17 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; -import org.apache.tajo.*; +import org.apache.tajo.QueryId; +import org.apache.tajo.QueryIdFactory; +import org.apache.tajo.TajoIdProtos; +import org.apache.tajo.TajoProtos; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.NoSuchDatabaseException; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.ipc.ClientProtos.*; import org.apache.tajo.ipc.TajoMasterClientProtocol; @@ -278,9 +282,16 @@ public UpdateQueryResponse updateQuery(RpcController controller, QueryRequest re try { Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + QueryContext queryContext = new QueryContext(conf, session); + if (queryContext.getCurrentDatabase() == null) { + for (Map.Entry e : queryContext.getAllKeyValus().entrySet()) { + System.out.println(e.getKey() + "=" + e.getValue()); + } + } + UpdateQueryResponse.Builder builder = UpdateQueryResponse.newBuilder(); try { - context.getGlobalEngine().updateQuery(session, request.getQuery(), request.getIsJson()); + context.getGlobalEngine().updateQuery(queryContext, request.getQuery(), request.getIsJson()); builder.setResultCode(ResultCode.OK); return builder.build(); } catch (Exception e) { @@ -539,7 +550,9 @@ public GetClusterInfoResponse getClusterInfo(RpcController controller, public BoolProto createDatabase(RpcController controller, SessionedStringProto request) throws ServiceException { try { Session session = context.getSessionManager().getSession(request.getSessionId().getId()); - if (context.getGlobalEngine().createDatabase(session, request.getValue(), null, false)) { + QueryContext queryContext = new QueryContext(conf, session); + + if (context.getGlobalEngine().createDatabase(queryContext, request.getValue(), null, false)) { return BOOL_TRUE; } else { return BOOL_FALSE; @@ -567,8 +580,9 @@ public BoolProto existDatabase(RpcController controller, SessionedStringProto re public BoolProto dropDatabase(RpcController controller, SessionedStringProto request) throws ServiceException { try { Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + QueryContext queryContext = new QueryContext(conf, session); - if (context.getGlobalEngine().dropDatabase(session, request.getValue(), false)) { + if (context.getGlobalEngine().dropDatabase(queryContext, request.getValue(), false)) { return BOOL_TRUE; } else { return BOOL_FALSE; @@ -605,6 +619,10 @@ public BoolProto existTable(RpcController controller, SessionedStringProto reque tableName = request.getValue(); } + if (databaseName == null) { + System.out.println("A"); + } + if (catalog.existsTable(databaseName, tableName)) { return BOOL_TRUE; } else { @@ -672,6 +690,7 @@ public TableResponse createExternalTable(RpcController controller, CreateTableRe throws ServiceException { try { Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + QueryContext queryContext = new QueryContext(conf, session); Path path = new Path(request.getPath()); FileSystem fs = path.getFileSystem(conf); @@ -689,7 +708,7 @@ public TableResponse createExternalTable(RpcController controller, CreateTableRe TableDesc desc; try { - desc = context.getGlobalEngine().createTableOnPath(session, request.getName(), schema, + desc = context.getGlobalEngine().createTableOnPath(queryContext, request.getName(), schema, meta, path, true, partitionDesc, false); } catch (Exception e) { return TableResponse.newBuilder() @@ -715,7 +734,9 @@ public TableResponse createExternalTable(RpcController controller, CreateTableRe public BoolProto dropTable(RpcController controller, DropTableRequest dropTable) throws ServiceException { try { Session session = context.getSessionManager().getSession(dropTable.getSessionId().getId()); - context.getGlobalEngine().dropTable(session, dropTable.getName(), false, dropTable.getPurge()); + QueryContext queryContext = new QueryContext(conf, session); + + context.getGlobalEngine().dropTable(queryContext, dropTable.getName(), false, dropTable.getPurge()); return BOOL_TRUE; } catch (Throwable t) { throw new ServiceException(t); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TaskSchedulerFactory.java b/tajo-core/src/main/java/org/apache/tajo/master/TaskSchedulerFactory.java index 520ecd32d0..744e968d72 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TaskSchedulerFactory.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TaskSchedulerFactory.java @@ -23,6 +23,7 @@ import org.apache.tajo.master.querymaster.SubQuery; import java.io.IOException; +import java.io.OutputStream; import java.lang.reflect.Constructor; import java.util.Map; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index 9156b09024..2e75873a64 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryId; +import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.catalog.CatalogService; @@ -636,7 +637,7 @@ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryCo SubQuery lastStage = query.getSubQuery(finalExecBlockId); TableMeta meta = lastStage.getTableMeta(); - String nullChar = queryContext.get(ConfVars.$CSVFILE_NULL.varname, ConfVars.$CSVFILE_NULL.defaultVal); + String nullChar = queryContext.get(SessionVars.NULL_CHAR); meta.putOption(StorageConstants.CSVFILE_NULL, nullChar); TableStats stats = lastStage.getResultStats(); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index 071e5d4f5d..5885a1d9d3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -328,8 +328,8 @@ public synchronized void startQuery() { LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(systemConf); Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class); - LogicalPlan plan = planner.createPlan(session, expr); - optimizer.optimize(session, plan); + LogicalPlan plan = planner.createPlan(queryContext, expr); + optimizer.optimize(queryContext, plan); GlobalEngine.DistributedQueryHookManager hookManager = new GlobalEngine.DistributedQueryHookManager(); hookManager.addHook(new GlobalEngine.InsertHook()); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index dd8d46f07c..899db07d5f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -24,12 +24,12 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.ExecutionBlockId; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.PlanningException; import org.apache.tajo.engine.planner.RangePartitionAlgorithm; @@ -39,7 +39,6 @@ import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.utils.TupleUtil; import org.apache.tajo.exception.InternalException; import org.apache.tajo.ipc.TajoWorkerProtocol; @@ -61,9 +60,7 @@ import java.util.*; import java.util.Map.Entry; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.HASH_SHUFFLE; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.RANGE_SHUFFLE; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.SCATTERED_HASH_SHUFFLE; +import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.*; /** * Repartitioner creates non-leaf tasks and shuffles intermediate data. @@ -375,7 +372,7 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster // Getting the desire number of join tasks according to the volumn // of a larger table int largerIdx = stats[0] >= stats[1] ? 0 : 1; - int desireJoinTaskVolumn = subQuery.getMasterPlan().getContext().getInt(ConfVars.DIST_QUERY_JOIN_TASK_VOLUME); + int desireJoinTaskVolumn = subQuery.getMasterPlan().getContext().getInt(SessionVars.JOIN_TASK_INPUT_SIZE); // calculate the number of tasks according to the data size int mb = (int) Math.ceil((double) stats[largerIdx] / 1048576); @@ -765,17 +762,17 @@ public static void scheduleHashShuffledFetches(TaskSchedulerContext schedulerCon // Scattered hash shuffle hashes the key columns and groups the hash keys associated with // the same hash key. Then, if the volume of a group is larger - // than DIST_QUERY_TABLE_PARTITION_VOLUME, it divides the group into more than two sub groups - // according to DIST_QUERY_TABLE_PARTITION_VOLUME (default size = 256MB). + // than $DIST_QUERY_TABLE_PARTITION_VOLUME, it divides the group into more than two sub groups + // according to $DIST_QUERY_TABLE_PARTITION_VOLUME (default size = 256MB). // As a result, each group size always becomes the less than or equal - // to DIST_QUERY_TABLE_PARTITION_VOLUME. Finally, each subgroup is assigned to a query unit. + // to $DIST_QUERY_TABLE_PARTITION_VOLUME. Finally, each subgroup is assigned to a query unit. // It is usually used for writing partitioned tables. public static void scheduleScatteredHashShuffleFetches(TaskSchedulerContext schedulerContext, SubQuery subQuery, Map> intermediates, String tableName) { int i = 0; long splitVolume = StorageUnit.MB * - subQuery.getMasterPlan().getContext().getLong(ConfVars.DIST_QUERY_TABLE_PARTITION_VOLUME); + subQuery.getMasterPlan().getContext().getLong(SessionVars.TABLE_PARTITION_WRITE_PER_SHUFFLE_SIZE); long sumNumBytes = 0L; Map> fetches = new HashMap>(); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index 06062afcd8..15aa507e89 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -741,7 +741,7 @@ public static int calculateShuffleOutputNum(SubQuery subQuery, DataChannel chann LOG.info(subQuery.getId() + ", Bigger Table's volume is approximately " + mb + " MB"); int taskNum = (int) Math.ceil((double) mb / - conf.getIntVar(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME)); + conf.getIntVar(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME)); int totalMem = getClusterTotalMemory(subQuery); LOG.info(subQuery.getId() + ", Total memory of cluster is " + totalMem + " MB"); @@ -749,8 +749,8 @@ public static int calculateShuffleOutputNum(SubQuery subQuery, DataChannel chann // determine the number of task taskNum = Math.min(taskNum, slots); - if (conf.getIntVar(ConfVars.$TESTCASE_MIN_TASK_NUM) > 0) { - taskNum = conf.getIntVar(ConfVars.$TESTCASE_MIN_TASK_NUM); + if (conf.getIntVar(ConfVars.$TEST_MIN_TASK_NUM) > 0) { + taskNum = conf.getIntVar(ConfVars.$TEST_MIN_TASK_NUM); LOG.warn("!!!!! TESTCASE MODE !!!!!"); } @@ -794,7 +794,7 @@ public static int calculateShuffleOutputNum(SubQuery subQuery, DataChannel chann LOG.info(subQuery.getId() + ", Table's volume is approximately " + mb + " MB"); // determine the number of task int taskNumBySize = (int) Math.ceil((double) mb / - conf.getIntVar(ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME)); + conf.getIntVar(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME)); int totalMem = getClusterTotalMemory(subQuery); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java b/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java index a67b6c8a85..cf4abcbeb5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java @@ -19,6 +19,7 @@ package org.apache.tajo.master.session; import com.google.common.collect.ImmutableMap; +import org.apache.tajo.SessionVars; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.common.ProtoObject; @@ -30,17 +31,21 @@ public class Session implements SessionConstants, ProtoObject { private final String sessionId; private final String userName; + private String currentDatabase; private final Map sessionVariables; // transient status private volatile long lastAccessTime; - private volatile String currentDatabase; public Session(String sessionId, String userName, String databaseName) { this.sessionId = sessionId; this.userName = userName; + this.currentDatabase = databaseName; this.lastAccessTime = System.currentTimeMillis(); + this.sessionVariables = new HashMap(); + sessionVariables.put(SessionVars.SESSION_ID.keyname(), sessionId); + sessionVariables.put(SessionVars.USERNAME.keyname(), userName); selectDatabase(databaseName); } @@ -85,16 +90,6 @@ public String getVariable(String name) throws NoSuchSessionVariableException { } } - public String getVariable(String name, String defaultValue) { - synchronized (sessionVariables) { - if (sessionVariables.containsKey(name)) { - return sessionVariables.get(name); - } else { - return defaultValue; - } - } - } - public void removeVariable(String name) { synchronized (sessionVariables) { sessionVariables.remove(name); @@ -103,32 +98,39 @@ public void removeVariable(String name) { public synchronized Map getAllVariables() { synchronized (sessionVariables) { + sessionVariables.put(SessionVars.SESSION_ID.keyname(), sessionId); + sessionVariables.put(SessionVars.USERNAME.keyname(), userName); + sessionVariables.put(SessionVars.SESSION_LAST_ACCESS_TIME.keyname(), String.valueOf(lastAccessTime)); + sessionVariables.put(SessionVars.CURRENT_DATABASE.keyname(), currentDatabase); return ImmutableMap.copyOf(sessionVariables); } } - public void selectDatabase(String databaseName) { + public synchronized void selectDatabase(String databaseName) { this.currentDatabase = databaseName; } - public String getCurrentDatabase() { - return this.currentDatabase; + public synchronized String getCurrentDatabase() { + return currentDatabase; } @Override public SessionProto getProto() { SessionProto.Builder builder = SessionProto.newBuilder(); - builder.setSessionId(sessionId); - builder.setUsername(userName); - builder.setCurrentDatabase(currentDatabase); + builder.setSessionId(getSessionId()); + builder.setUsername(getUserName()); + builder.setCurrentDatabase(getCurrentDatabase()); builder.setLastAccessTime(lastAccessTime); KeyValueSet variables = new KeyValueSet(); - variables.putAll(this.sessionVariables); - builder.setVariables(variables.getProto()); - return builder.build(); + + synchronized (sessionVariables) { + variables.putAll(this.sessionVariables); + builder.setVariables(variables.getProto()); + return builder.build(); + } } public String toString() { - return "user=" + userName + ",id=" + sessionId; + return "user=" + getUserName() + ",id=" + getSessionId() +",last_atime=" + getLastAccessTime(); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java index a5783cc4b9..4192ca0a40 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java @@ -55,7 +55,7 @@ public TajoSystemMetrics(TajoConf tajoConf, String metricsGroupName, String host this.hostAndPort = hostAndPort; try { - this.metricsPropertyFileName = tajoConf.getVar(TajoConf.ConfVars.$METRICS_PROPERTY_FILENAME); + this.metricsPropertyFileName = tajoConf.getVar(TajoConf.ConfVars.METRICS_PROPERTY_FILENAME); this.metricsProps = new PropertiesConfiguration(metricsPropertyFileName); this.metricsProps.addConfigurationListener(new MetricsReloadListener()); FileChangedReloadingStrategy reloadingStrategy = new FileChangedReloadingStrategy(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index eb769260a7..68ee64f301 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -537,7 +537,7 @@ public TaskHistory createTaskHistory() { FetcherHistoryProto.Builder builder = FetcherHistoryProto.newBuilder(); for (Fetcher fetcher : fetcherRunners) { // TODO store the fetcher histories - if (systemConf.getBoolVar(TajoConf.ConfVars.$TAJO_DEBUG)) { + if (systemConf.getBoolVar(TajoConf.ConfVars.$DEBUG_ENABLED)) { builder.setStartTime(fetcher.getStartTime()); builder.setFinishTime(fetcher.getFinishTime()); builder.setFileLength(fetcher.getFileLen()); diff --git a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java index 84522e1942..271ba702dd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java +++ b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java @@ -31,6 +31,7 @@ import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.MasterPlan; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.session.Session; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TajoIdUtils; @@ -69,6 +70,10 @@ public static Session createDummySession() { return new Session(UUID.randomUUID().toString(), dummyUserInfo.getUserName(), TajoConstants.DEFAULT_DATABASE_NAME); } + public static QueryContext createDummyContext(TajoConf conf) { + return new QueryContext(conf, createDummySession()); + } + /** * for test * @return The generated QueryId diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 9d157320d9..8cd1bff7ff 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -214,7 +214,7 @@ private void init() { } catch (ServiceException e) { e.printStackTrace(); } - testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "false"); + testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false"); } protected TajoClient getClient() { diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index a34be4e11d..5d480d2fa8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -176,7 +176,8 @@ public final void testSessionVariables() throws IOException, ServiceException, I String key = prefixName + i; String val = prefixValue + i; - assertEquals(i, client.getAllSessionVariables().size()); + // Basically, + assertEquals(i + 4, client.getAllSessionVariables().size()); assertFalse(client.getAllSessionVariables().containsKey(key)); assertFalse(client.existSessionVariable(key)); @@ -184,7 +185,7 @@ public final void testSessionVariables() throws IOException, ServiceException, I map.put(key, val); client.updateSessionVariables(map); - assertEquals(i + 1, client.getAllSessionVariables().size()); + assertEquals(i + 5, client.getAllSessionVariables().size()); assertTrue(client.getAllSessionVariables().containsKey(key)); assertTrue(client.existSessionVariable(key)); } @@ -690,7 +691,7 @@ public void testSetCvsNull() throws Exception { TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); Map variables = new HashMap(); - variables.put(ConfVars.$CSVFILE_NULL.varname, "\\\\T"); + variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T"); client.updateSessionVariables(variables); TajoResultSet res = (TajoResultSet)client.executeQueryAndGetResult(sql); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index ad80ddfafe..7f9436a482 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -32,6 +32,7 @@ import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; @@ -107,22 +108,24 @@ private static Target[] getRawTargets(String query, boolean condition) throws Pl InvalidStatementException { Session session = LocalTajoTestingUtility.createDummySession(); + QueryContext context = new QueryContext(util.getConfiguration(), session); + List parsedResults = SimpleParser.parseScript(query); if (parsedResults.size() > 1) { throw new RuntimeException("this query includes two or more statements."); } Expr expr = analyzer.parse(parsedResults.get(0).getHistoryStatement()); VerificationState state = new VerificationState(); - preLogicalPlanVerifier.verify(session, state, expr); + preLogicalPlanVerifier.verify(context, state, expr); if (state.getErrorMessages().size() > 0) { if (!condition && state.getErrorMessages().size() > 0) { throw new PlanningException(state.getErrorMessages().get(0)); } assertFalse(state.getErrorMessages().get(0), true); } - LogicalPlan plan = planner.createPlan(session, expr, true); + LogicalPlan plan = planner.createPlan(context, expr, true); optimizer.optimize(plan); - annotatedPlanVerifier.verify(session, state, plan); + annotatedPlanVerifier.verify(context, state, plan); if (state.getErrorMessages().size() > 0) { assertFalse(state.getErrorMessages().get(0), true); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index 7bb619d815..409b7ec823 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -40,6 +40,7 @@ import org.apache.tajo.engine.planner.logical.GroupbyNode; import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InternalException; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; @@ -67,7 +68,7 @@ public class TestEvalTreeUtil { static EvalNode expr3; static SQLAnalyzer analyzer; static LogicalPlanner planner; - static Session session = LocalTajoTestingUtility.createDummySession(); + static QueryContext defaultContext; public static class TestSum extends GeneralFunction { private Integer x; @@ -123,6 +124,8 @@ public static void setUp() throws Exception { "select name, score, age from people where test_sum(score * age, 50)", // 2 }; + defaultContext = LocalTajoTestingUtility.createDummyContext(util.getConfiguration()); + expr1 = getRootSelection(QUERIES[0]); expr2 = getRootSelection(QUERIES[1]); expr3 = getRootSelection(QUERIES[2]); @@ -137,7 +140,7 @@ public static void tearDown() throws Exception { Expr expr = analyzer.parse(query); LogicalPlan plan = null; try { - plan = planner.createPlan(session, expr); + plan = planner.createPlan(defaultContext, expr); } catch (PlanningException e) { e.printStackTrace(); } @@ -149,7 +152,7 @@ public static EvalNode getRootSelection(String query) throws PlanningException { Expr block = analyzer.parse(query); LogicalPlan plan = null; try { - plan = planner.createPlan(session, block); + plan = planner.createPlan(defaultContext, block); } catch (PlanningException e) { e.printStackTrace(); } @@ -225,7 +228,7 @@ public final void testGetSchemaFromTargets() throws InternalException { @Test public final void testGetContainExprs() throws CloneNotSupportedException, PlanningException { Expr expr = analyzer.parse(QUERIES[1]); - LogicalPlan plan = planner.createPlan(session, expr, true); + LogicalPlan plan = planner.createPlan(defaultContext, expr, true); Target [] targets = plan.getRootBlock().getRawTargets(); Column col1 = new Column("default.people.score", TajoDataTypes.Type.INT4); Collection exprs = @@ -303,7 +306,7 @@ public final void testSimplify() throws PlanningException { assertTrue(7.0d == node.eval(null, null).asFloat8()); Expr expr = analyzer.parse(QUERIES[1]); - LogicalPlan plan = planner.createPlan(session, expr, true); + LogicalPlan plan = planner.createPlan(defaultContext, expr, true); targets = plan.getRootBlock().getRawTargets(); Column col1 = new Column("default.people.score", TajoDataTypes.Type.INT4); Collection exprs = @@ -343,7 +346,7 @@ public final void testTranspose() throws PlanningException { public final void testFindDistinctAggFunctions() throws PlanningException { String query = "select sum(score) + max(age) from people"; Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); GroupbyNode groupByNode = plan.getRootBlock().getNode(NodeType.GROUP_BY); EvalNode [] aggEvals = groupByNode.getAggFunctions(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java index 773382dc2b..d7dd78a7d1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java @@ -29,6 +29,7 @@ import org.apache.tajo.engine.function.builtin.SumInt; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; import org.apache.tajo.util.CommonTestingUtil; @@ -48,7 +49,7 @@ public class TestLogicalOptimizer { private static SQLAnalyzer sqlAnalyzer; private static LogicalPlanner planner; private static LogicalOptimizer optimizer; - private static Session session = LocalTajoTestingUtility.createDummySession(); + private static QueryContext defaultContext; @BeforeClass public static void setUp() throws Exception { @@ -101,6 +102,8 @@ public static void setUp() throws Exception { sqlAnalyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); optimizer = new LogicalOptimizer(util.getConfiguration()); + + defaultContext = LocalTajoTestingUtility.createDummyContext(util.getConfiguration()); } @AfterClass @@ -121,7 +124,7 @@ public static void tearDown() throws Exception { public final void testProjectionPushWithNaturalJoin() throws PlanningException, CloneNotSupportedException { // two relations Expr expr = sqlAnalyzer.parse(QUERIES[4]); - LogicalPlan newPlan = planner.createPlan(session, expr); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); LogicalNode plan = newPlan.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; @@ -148,7 +151,7 @@ public final void testProjectionPushWithNaturalJoin() throws PlanningException, public final void testProjectionPushWithInnerJoin() throws PlanningException { // two relations Expr expr = sqlAnalyzer.parse(QUERIES[5]); - LogicalPlan newPlan = planner.createPlan(session, expr); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); optimizer.optimize(newPlan); } @@ -156,7 +159,7 @@ public final void testProjectionPushWithInnerJoin() throws PlanningException { public final void testProjectionPush() throws CloneNotSupportedException, PlanningException { // two relations Expr expr = sqlAnalyzer.parse(QUERIES[2]); - LogicalPlan newPlan = planner.createPlan(session, expr); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); LogicalNode plan = newPlan.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); @@ -178,7 +181,7 @@ public final void testProjectionPush() throws CloneNotSupportedException, Planni @Test public final void testOptimizeWithGroupBy() throws CloneNotSupportedException, PlanningException { Expr expr = sqlAnalyzer.parse(QUERIES[3]); - LogicalPlan newPlan = planner.createPlan(session, expr); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); LogicalNode plan = newPlan.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); @@ -205,7 +208,7 @@ public final void testOptimizeWithGroupBy() throws CloneNotSupportedException, P public final void testPushable() throws CloneNotSupportedException, PlanningException { // two relations Expr expr = sqlAnalyzer.parse(QUERIES[0]); - LogicalPlan newPlan = planner.createPlan(session, expr); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); LogicalNode plan = newPlan.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); @@ -236,7 +239,7 @@ public final void testPushable() throws CloneNotSupportedException, PlanningExce // Scan Pushable Test expr = sqlAnalyzer.parse(QUERIES[1]); - newPlan = planner.createPlan(session, expr); + newPlan = planner.createPlan(defaultContext, expr); plan = newPlan.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); @@ -258,7 +261,7 @@ public final void testPushable() throws CloneNotSupportedException, PlanningExce @Test public final void testInsertInto() throws CloneNotSupportedException, PlanningException { Expr expr = sqlAnalyzer.parse(TestLogicalPlanner.insertStatements[0]); - LogicalPlan newPlan = planner.createPlan(session, expr); + LogicalPlan newPlan = planner.createPlan(defaultContext, expr); optimizer.optimize(newPlan); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java index 2f1e0f9ef1..61770f824e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java @@ -88,7 +88,7 @@ public static void tearDown() { @Test public final void testQueryBlockGraph() { - LogicalPlan plan = new LogicalPlan(LocalTajoTestingUtility.createDummySession().getCurrentDatabase(), planner); + LogicalPlan plan = new LogicalPlan(planner); LogicalPlan.QueryBlock root = plan.newAndGetBlock(LogicalPlan.ROOT_BLOCK); LogicalPlan.QueryBlock new1 = plan.newQueryBlock(); LogicalPlan.QueryBlock new2 = plan.newQueryBlock(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 6d0f7e01f9..73f72be43c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -37,6 +37,7 @@ import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; import org.apache.tajo.util.CommonTestingUtil; @@ -154,8 +155,10 @@ public static void tearDown() throws Exception { @Test public final void testSingleRelation() throws CloneNotSupportedException, PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(QUERIES[0]); - LogicalPlan planNode = planner.createPlan(session, expr); + LogicalPlan planNode = planner.createPlan(qc, expr); LogicalNode plan = planNode.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); TestLogicalNode.testCloneLogicalNode(plan); @@ -186,9 +189,11 @@ public static void assertSchema(Schema expected, Schema schema) { @Test public final void testImplicityJoinPlan() throws CloneNotSupportedException, PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + // two relations Expr expr = sqlAnalyzer.parse(QUERIES[1]); - LogicalPlan planNode = planner.createPlan(session, expr); + LogicalPlan planNode = planner.createPlan(qc, expr); LogicalNode plan = planNode.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); @@ -221,7 +226,7 @@ public final void testImplicityJoinPlan() throws CloneNotSupportedException, Pla // three relations expr = sqlAnalyzer.parse(QUERIES[2]); - plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); TestLogicalNode.testCloneLogicalNode(plan); @@ -273,9 +278,10 @@ public final void testImplicityJoinPlan() throws CloneNotSupportedException, Pla @Test public final void testNaturalJoinPlan() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); // two relations Expr context = sqlAnalyzer.parse(JOINS[0]); - LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, context).getRootBlock().getRoot(); testJsonSerDerObject(plan); assertSchema(expectedJoinSchema, plan.getOutSchema()); @@ -304,9 +310,10 @@ public final void testNaturalJoinPlan() throws PlanningException { @Test public final void testInnerJoinPlan() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); // two relations Expr expr = sqlAnalyzer.parse(JOINS[1]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(qc, expr); LogicalNode root = plan.getRootBlock().getRoot(); testJsonSerDerObject(root); assertSchema(expectedJoinSchema, root.getOutSchema()); @@ -336,9 +343,11 @@ public final void testInnerJoinPlan() throws PlanningException { @Test public final void testOuterJoinPlan() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + // two relations Expr expr = sqlAnalyzer.parse(JOINS[2]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); assertSchema(expectedJoinSchema, plan.getOutSchema()); @@ -369,9 +378,11 @@ public final void testOuterJoinPlan() throws PlanningException { @Test public final void testGroupby() throws CloneNotSupportedException, PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + // without 'having clause' Expr context = sqlAnalyzer.parse(QUERIES[7]); - LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, context).getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; @@ -380,7 +391,7 @@ public final void testGroupby() throws CloneNotSupportedException, PlanningExcep // with having clause context = sqlAnalyzer.parse(QUERIES[3]); - plan = planner.createPlan(session, context).getRootBlock().getRoot(); + plan = planner.createPlan(qc, context).getRootBlock().getRoot(); TestLogicalNode.testCloneLogicalNode(plan); assertEquals(NodeType.ROOT, plan.getType()); @@ -411,7 +422,8 @@ public final void testGroupby() throws CloneNotSupportedException, PlanningExcep public final void testMultipleJoin() throws IOException, PlanningException { Expr expr = sqlAnalyzer.parse( FileUtil.readTextFile(new File("src/test/resources/queries/TestJoinQuery/testTPCHQ2Join.sql"))); - LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr).getRootBlock().getRoot(); + QueryContext qc = new QueryContext(util.getConfiguration(), session); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); Schema expected = tpch.getOutSchema("q2"); assertSchema(expected, plan.getOutSchema()); @@ -448,8 +460,8 @@ private final void findJoinQual(EvalNode evalNode, Map qual qualMap.put(entry.getKey(), Boolean.TRUE); } } else if (rightType == EvalType.ROW_CONSTANT) { - RowConstantEval rightField = (RowConstantEval)qual.getRightExpr(); - RowConstantEval rightJoinField = (RowConstantEval)entry.getKey().getRightExpr(); + RowConstantEval rightField = qual.getRightExpr(); + RowConstantEval rightJoinField = entry.getKey().getRightExpr(); if (leftJoinField.getColumnRef().getQualifiedName().equals(leftField.getColumnRef().getQualifiedName())) { assertEquals(rightField.getValues().length, rightJoinField.getValues().length); @@ -469,8 +481,9 @@ public final void testJoinWithMultipleJoinQual1() throws IOException, PlanningEx Expr expr = sqlAnalyzer.parse( FileUtil.readTextFile(new File ("src/test/resources/queries/TestJoinQuery/testJoinWithMultipleJoinQual1.sql"))); + QueryContext qc = new QueryContext(util.getConfiguration(), session); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),expr); + LogicalPlan plan = planner.createPlan(qc, expr); LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); @@ -510,8 +523,9 @@ public final void testJoinWithMultipleJoinQual2() throws IOException, PlanningEx Expr expr = sqlAnalyzer.parse( FileUtil.readTextFile(new File ("src/test/resources/queries/TestJoinQuery/testJoinWithMultipleJoinQual2.sql"))); + QueryContext qc = new QueryContext(util.getConfiguration(), session); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),expr); + LogicalPlan plan = planner.createPlan(qc,expr); LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); @@ -550,8 +564,9 @@ public final void testJoinWithMultipleJoinQual3() throws IOException, PlanningEx Expr expr = sqlAnalyzer.parse( FileUtil.readTextFile(new File ("src/test/resources/queries/TestJoinQuery/testJoinWithMultipleJoinQual3.sql"))); + QueryContext qc = new QueryContext(util.getConfiguration(), session); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),expr); + LogicalPlan plan = planner.createPlan(qc, expr); LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); @@ -596,8 +611,9 @@ public final void testJoinWithMultipleJoinQual4() throws IOException, PlanningEx Expr expr = sqlAnalyzer.parse( FileUtil.readTextFile(new File ("src/test/resources/queries/TestJoinQuery/testJoinWithMultipleJoinQual4.sql"))); + QueryContext qc = new QueryContext(util.getConfiguration(), session); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(),expr); + LogicalPlan plan = planner.createPlan(qc, expr); LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); @@ -683,8 +699,11 @@ static void testQuery7(LogicalNode plan) { @Test public final void testStoreTable() throws CloneNotSupportedException, PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr context = sqlAnalyzer.parse(QUERIES[8]); - LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot(); + + LogicalNode plan = planner.createPlan(qc, context).getRootBlock().getRoot(); TestLogicalNode.testCloneLogicalNode(plan); testJsonSerDerObject(plan); @@ -698,8 +717,11 @@ public final void testStoreTable() throws CloneNotSupportedException, PlanningEx @Test public final void testOrderBy() throws CloneNotSupportedException, PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(QUERIES[4]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); TestLogicalNode.testCloneLogicalNode(plan); @@ -725,8 +747,11 @@ public final void testOrderBy() throws CloneNotSupportedException, PlanningExcep @Test public final void testLimit() throws CloneNotSupportedException, PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(QUERIES[12]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); TestLogicalNode.testCloneLogicalNode(plan); @@ -744,8 +769,10 @@ public final void testLimit() throws CloneNotSupportedException, PlanningExcepti @Test public final void testSPJPush() throws CloneNotSupportedException, PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(QUERIES[5]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); TestLogicalNode.testCloneLogicalNode(plan); @@ -764,16 +791,20 @@ public final void testSPJPush() throws CloneNotSupportedException, PlanningExcep @Test public final void testSPJ() throws CloneNotSupportedException, PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(QUERIES[6]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); TestLogicalNode.testCloneLogicalNode(plan); } @Test public final void testJson() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(QUERIES[9]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); String json = plan.toJson(); @@ -792,9 +823,11 @@ public final void testJson() throws PlanningException { @Test public final void testVisitor() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + // two relations Expr expr = sqlAnalyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); TestVisitor vis = new TestVisitor(); plan.postOrder(vis); @@ -817,8 +850,10 @@ public void visit(LogicalNode node) { @Test public final void testExprNode() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(QUERIES[10]); - LogicalPlan rootNode = planner.createPlan(session, expr); + LogicalPlan rootNode = planner.createPlan(qc, expr); LogicalNode plan = rootNode.getRootBlock().getRoot(); testJsonSerDerObject(plan); assertEquals(NodeType.ROOT, plan.getType()); @@ -837,8 +872,10 @@ public final void testExprNode() throws PlanningException { @Test public final void testAsterisk() throws CloneNotSupportedException, PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(QUERIES[13]); - LogicalPlan planNode = planner.createPlan(session, expr); + LogicalPlan planNode = planner.createPlan(qc, expr); LogicalNode plan = planNode.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); TestLogicalNode.testCloneLogicalNode(plan); @@ -865,8 +902,10 @@ public final void testAsterisk() throws CloneNotSupportedException, PlanningExce @Test public final void testAlias1() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(ALIAS[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); LogicalRootNode root = (LogicalRootNode) plan; testJsonSerDerObject(root); @@ -878,7 +917,7 @@ public final void testAlias1() throws PlanningException { assertEquals("total", col.getSimpleName()); expr = sqlAnalyzer.parse(ALIAS[1]); - plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); root = (LogicalRootNode) plan; finalSchema = root.getOutSchema(); @@ -891,8 +930,10 @@ public final void testAlias1() throws PlanningException { @Test public final void testAlias2() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(ALIAS[1]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); LogicalRootNode root = (LogicalRootNode) plan; testJsonSerDerObject(root); @@ -910,8 +951,10 @@ public final void testAlias2() throws PlanningException { @Test public final void testCreateTableDef() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(CREATE_TABLE[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); LogicalRootNode root = (LogicalRootNode) plan; testJsonSerDerObject(root); assertEquals(NodeType.CREATE_TABLE, root.getChild().getType()); @@ -994,8 +1037,10 @@ public final void testGenerateCuboids() { @Test public final void testSetPlan() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(setStatements[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; @@ -1013,8 +1058,10 @@ public final void testSetPlan() throws PlanningException { @Test public void testSetQualifier() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr context = sqlAnalyzer.parse(setQualifiers[0]); - LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(qc, context).getRootBlock().getRoot(); testJsonSerDerObject(plan); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; @@ -1023,7 +1070,7 @@ public void testSetQualifier() throws PlanningException { assertEquals(NodeType.SCAN, projectionNode.getChild().getType()); context = sqlAnalyzer.parse(setQualifiers[1]); - plan = planner.createPlan(session, context).getRootBlock().getRoot(); + plan = planner.createPlan(qc, context).getRootBlock().getRoot(); testJsonSerDerObject(plan); assertEquals(NodeType.ROOT, plan.getType()); root = (LogicalRootNode) plan; @@ -1032,7 +1079,7 @@ public void testSetQualifier() throws PlanningException { assertEquals(NodeType.GROUP_BY, projectionNode.getChild().getType()); context = sqlAnalyzer.parse(setQualifiers[2]); - plan = planner.createPlan(session, context).getRootBlock().getRoot(); + plan = planner.createPlan(qc, context).getRootBlock().getRoot(); testJsonSerDerObject(plan); root = (LogicalRootNode) plan; assertEquals(NodeType.PROJECTION, root.getChild().getType()); @@ -1063,8 +1110,10 @@ public void testJsonSerDerObject(LogicalNode rootNode) { @Test public final void testInsertInto0() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(insertStatements[0]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(qc, expr); assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); @@ -1074,8 +1123,10 @@ public final void testInsertInto0() throws PlanningException { @Test public final void testInsertInto1() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(insertStatements[1]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(qc, expr); assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); @@ -1084,8 +1135,10 @@ public final void testInsertInto1() throws PlanningException { @Test public final void testInsertInto2() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(insertStatements[2]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(qc, expr); assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); @@ -1097,8 +1150,10 @@ public final void testInsertInto2() throws PlanningException { @Test public final void testInsertInto3() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(insertStatements[3]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(qc, expr); assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); @@ -1107,8 +1162,10 @@ public final void testInsertInto3() throws PlanningException { @Test public final void testInsertInto4() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(insertStatements[4]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(qc, expr); assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertTrue(insertNode.isOverwrite()); @@ -1121,8 +1178,10 @@ public final void testInsertInto4() throws PlanningException { @Test public final void testInsertInto5() throws PlanningException { + QueryContext qc = new QueryContext(util.getConfiguration(), session); + Expr expr = sqlAnalyzer.parse(insertStatements[5]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(qc, expr); assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertTrue(insertNode.isOverwrite()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index 82e781834f..cf3840f24c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -52,7 +52,6 @@ public class TestPlannerUtil { private static CatalogService catalog; private static SQLAnalyzer analyzer; private static LogicalPlanner planner; - private static Session session = LocalTajoTestingUtility.createDummySession(); @BeforeClass public static void setUp() throws Exception { @@ -111,7 +110,8 @@ public static void tearDown() throws Exception { public final void testFindTopNode() throws CloneNotSupportedException, PlanningException { // two relations Expr expr = analyzer.parse(TestLogicalPlanner.QUERIES[1]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(util.getConfiguration()), + expr).getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index 1cf5295296..4beb5c55bc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -66,6 +66,7 @@ public class TestBroadcastJoinPlan { private TajoTestingCluster util; private CatalogService catalog; private SQLAnalyzer analyzer; + private QueryContext defaultContext; private Path testDir; private TableDesc smallTable1; @@ -79,8 +80,8 @@ public class TestBroadcastJoinPlan { public void setUp() throws Exception { util = new TajoTestingCluster(); conf = util.getConfiguration(); - conf.setLongVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD, 500 * 1024); - conf.setBoolVar(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO, true); + conf.setLongVar(TajoConf.ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, 500 * 1024); + conf.setBoolVar(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED, true); testDir = CommonTestingUtil.getTestDir(TEST_PATH); catalog = util.startCatalogCluster().getCatalog(); @@ -126,6 +127,7 @@ public void setUp() throws Exception { catalog.createTable(largeTable3); analyzer = new SQLAnalyzer(); + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); } private TableDesc makeTestData(String tableName, Schema schema, int dataSize) throws Exception { @@ -183,7 +185,7 @@ public final void testBroadcastJoin() throws IOException, PlanningException { LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -244,7 +246,7 @@ public final void testBroadcastJoinAllSmallTables() throws IOException, Planning LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -308,7 +310,7 @@ public final void testNotBroadcastJoinTwoLargeTable() throws IOException, Planni LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -336,7 +338,7 @@ public final void testTwoBroadcastJoin() throws IOException, PlanningException { LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -386,7 +388,7 @@ public final void testNotBroadcastJoinSubquery() throws IOException, PlanningExc LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -427,7 +429,7 @@ public final void testBroadcastJoinSubquery() throws IOException, PlanningExcept LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -483,7 +485,7 @@ public final void testLeftOuterJoinCase1() throws IOException, PlanningException LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -537,7 +539,7 @@ public final void testLeftOuterJoinCase2() throws IOException, PlanningException LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -620,7 +622,7 @@ public final void testLeftOuterJoinCase3() throws IOException, PlanningException LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -703,7 +705,7 @@ public final void testLeftOuterJoinCase4() throws IOException, PlanningException LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -762,7 +764,7 @@ public final void testLeftOuterJoinCase5() throws IOException, PlanningException LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -815,7 +817,7 @@ public final void testLeftOuterJoinCase6() throws IOException, PlanningException LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -907,7 +909,7 @@ public final void testInnerLeftOuterJoinCase1() throws IOException, PlanningExce LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); @@ -972,7 +974,7 @@ public final void testBroadcastCasebyCase1() throws IOException, PlanningExcepti LogicalPlanner planner = new LogicalPlanner(catalog); LogicalOptimizer optimizer = new LogicalOptimizer(conf); Expr expr = analyzer.parse(query); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); optimizer.optimize(plan); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index 43caec8506..3fecabd5a6 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -141,7 +141,8 @@ public void tearDown() throws Exception { @Test public final void testBNLCrossJoin() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), + expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN); @@ -174,7 +175,7 @@ public final void testBNLCrossJoin() throws IOException, PlanningException { @Test public final void testBNLInnerJoin() throws IOException, PlanningException { Expr context = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), + LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context).getRootBlock().getRoot(); FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index e7c10a137d..f817776b99 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -172,7 +172,7 @@ public void testEqual() throws Exception { TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); Expr expr = analyzer.parse(QUERY); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = optimizer.optimize(plan); TmpPlanner phyPlanner = new TmpPlanner(conf, sm); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 9236c97886..e7aac3c79c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -125,7 +125,7 @@ public final void testNext() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = plan.getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 8addcdb7f0..a9b2bb8465 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -61,7 +61,7 @@ public class TestFullOuterHashJoinExec { private LogicalPlanner planner; private AbstractStorageManager sm; private Path testDir; - private static Session session = LocalTajoTestingUtility.createDummySession(); + private QueryContext defaultContext; private TableDesc dep3; private TableDesc job3; @@ -234,11 +234,10 @@ public void setUp() throws Exception { phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path); catalog.createTable(phone3); - - - analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); + + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); } @After @@ -260,7 +259,7 @@ public void tearDown() throws Exception { @Test public final void testFullOuterHashJoinExec0() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); @@ -297,7 +296,7 @@ public final void testFullOuterHashJoinExec0() throws IOException, PlanningExcep @Test public final void testFullOuterHashJoinExec1() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); @@ -333,7 +332,7 @@ public final void testFullOuterHashJoinExec1() throws IOException, PlanningExcep @Test public final void testFullOuterHashJoinExec2() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[2]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); @@ -370,7 +369,7 @@ public final void testFullOuterHashJoinExec2() throws IOException, PlanningExcep @Test public final void testFullOuterHashJoinExec3() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[3]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 2b137cc9f5..1b9f7aaa0a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -36,7 +36,6 @@ import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -62,7 +61,7 @@ public class TestFullOuterMergeJoinExec { private LogicalPlanner planner; private AbstractStorageManager sm; private Path testDir; - private static final Session session = LocalTajoTestingUtility.createDummySession(); + private QueryContext defaultContext; private TableDesc dep3; private TableDesc dep4; @@ -279,6 +278,8 @@ public void setUp() throws Exception { analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); + + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); } @After @@ -304,7 +305,7 @@ public void tearDown() throws Exception { @Test public final void testFullOuterMergeJoin0() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -341,7 +342,7 @@ public final void testFullOuterMergeJoin0() throws IOException, PlanningExceptio @Test public final void testFullOuterMergeJoin1() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -378,7 +379,7 @@ public final void testFullOuterMergeJoin1() throws IOException, PlanningExceptio @Test public final void testFullOuterMergeJoin2() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[2]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -416,7 +417,7 @@ public final void testFullOuterMergeJoin2() throws IOException, PlanningExceptio @Test public final void testFullOuterMergeJoin3() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[3]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -456,7 +457,7 @@ public final void testFullOuterMergeJoin3() throws IOException, PlanningExceptio @Test public final void testFullOuterMergeJoin4() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[4]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -495,7 +496,7 @@ public final void testFullOuterMergeJoin4() throws IOException, PlanningExceptio @Test public final void testFullOuterMergeJoin5() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[5]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index d4c3c309f0..7055e44c17 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -158,7 +158,7 @@ public final void testHashAntiJoin() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); optimizer.optimize(plan); LogicalNode rootNode = plan.getRootBlock().getRoot(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index 8364c4120c..5bc05c2173 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -61,7 +61,7 @@ public class TestHashJoinExec { private LogicalPlanner planner; private AbstractStorageManager sm; private Path testDir; - private final Session session = LocalTajoTestingUtility.createDummySession(); + private QueryContext defaultContext; private TableDesc employee; private TableDesc people; @@ -126,6 +126,7 @@ public void setUp() throws Exception { catalog.createTable(people); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); } @After @@ -140,8 +141,9 @@ public void tearDown() throws Exception { @Test public final void testHashInnerJoin() throws IOException, PlanningException { + Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); @@ -182,7 +184,7 @@ public final void testHashInnerJoin() throws IOException, PlanningException { @Test public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); @@ -200,7 +202,7 @@ public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, P ctx.setEnforcer(enforcer); TajoConf localConf = new TajoConf(conf); - localConf.setLongVar(TajoConf.ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD, 100l); + localConf.setLongVar(TajoConf.ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, 100l); ctx.getQueryContext().setConf(localConf); PhysicalPlannerImpl phyPlanner = new PhysicalPlannerImpl(localConf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index f13c11625b..6373f232a4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -162,7 +162,7 @@ public final void testHashSemiJoin() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); optimizer.optimize(plan); LogicalNode rootNode = plan.getRootBlock().getRoot(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 96fb83add2..1bfa4abfff 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -62,7 +62,7 @@ public class TestLeftOuterHashJoinExec { private LogicalPlanner planner; private AbstractStorageManager sm; private Path testDir; - private Session session = LocalTajoTestingUtility.createDummySession(); + private QueryContext defaultContext; private TableDesc dep3; private TableDesc job3; @@ -239,6 +239,7 @@ public void setUp() throws Exception { analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); } @After @@ -262,7 +263,7 @@ public void tearDown() throws Exception { @Test public final void testLeftOuterHashJoinExec0() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN); @@ -307,7 +308,7 @@ public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningExce LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); @@ -348,7 +349,7 @@ public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningExce LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[2]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); @@ -390,7 +391,7 @@ public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningExce LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[3]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); @@ -432,7 +433,7 @@ public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningExce LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[4]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); @@ -444,12 +445,10 @@ public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningExce assertEquals(1, 0); } else{ - Tuple tuple; int count = 0; - int i = 1; exec.init(); - while ((tuple = exec.next()) != null) { + while (exec.next() != null) { //TODO check contents count = count + 1; } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index 66aab0ad0c..2a8d9f0108 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -36,7 +36,6 @@ import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -60,7 +59,7 @@ public class TestLeftOuterNLJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private static final Session session = LocalTajoTestingUtility.createDummySession(); + private QueryContext defaultContext; private AbstractStorageManager sm; private Path testDir; @@ -236,6 +235,8 @@ public void setUp() throws Exception { analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); + + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); } @After @@ -265,7 +266,7 @@ public final void testLeftOuterNLJoinExec0() throws IOException, PlanningExcepti LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); @@ -307,7 +308,7 @@ public final void testLeftOuterNLJoinExec1() throws IOException, PlanningExcepti LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); @@ -351,7 +352,7 @@ public final void testLeftOuter_NLJoinExec2() throws IOException, PlanningExcept LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[2]); - LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); @@ -396,7 +397,7 @@ public final void testLeftOuter_NLJoinExec3() throws IOException, PlanningExcept LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[3]); - LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); @@ -440,7 +441,7 @@ public final void testLeftOuter_NLJoinExec4() throws IOException, PlanningExcept LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[4]); - LogicalNode plan = planner.createPlan(session, context).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, context).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index 97820b39e3..ed19e4f5f7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -154,7 +154,7 @@ public void tearDown() throws Exception { @Test public final void testMergeInnerJoin() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[0]); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode root = plan.getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(root, NodeType.JOIN); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 397425970d..1120bbe1f7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -155,9 +155,9 @@ public final void testNLCrossJoin() throws IOException, PlanningException { TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); - Expr context = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), - context).getRootBlock().getRoot(); + Expr expr = analyzer.parse(QUERIES[0]); + LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), + expr).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); @@ -184,10 +184,9 @@ public final void testNLInnerJoin() throws IOException, PlanningException { TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), merged, workDir); ctx.setEnforcer(new Enforcer()); - Expr context = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), - context).getRootBlock().getRoot(); - //LogicalOptimizer.optimize(ctx, plan); + Expr expr = analyzer.parse(QUERIES[1]); + LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), + expr).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index a890164ccf..a184a9a0eb 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -85,6 +85,7 @@ public class TestPhysicalPlanner { private static AbstractStorageManager sm; private static Path testDir; private static Session session = LocalTajoTestingUtility.createDummySession(); + private static QueryContext defaultContext; private static TableDesc employee = null; private static TableDesc score = null; @@ -168,6 +169,7 @@ public static void setUp() throws Exception { planner = new LogicalPlanner(catalog); optimizer = new LogicalOptimizer(conf); + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null); } @@ -206,7 +208,7 @@ public final void testCreateScanPlan() throws IOException, PlanningException { new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); LogicalNode rootNode =plan.getRootBlock().getRoot(); optimizer.optimize(plan); @@ -237,7 +239,7 @@ public final void testCreateScanWithFilterPlan() throws IOException, PlanningExc new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[16]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); LogicalNode rootNode =plan.getRootBlock().getRoot(); optimizer.optimize(plan); @@ -266,7 +268,7 @@ public final void testGroupByPlan() throws IOException, PlanningException { new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[7]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); optimizer.optimize(plan); LogicalNode rootNode = plan.getRootBlock().getRoot(); @@ -298,7 +300,7 @@ public final void testHashGroupByPlanWithALLField() throws IOException, Planning new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[15]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); @@ -327,7 +329,7 @@ public final void testSortGroupByPlan() throws IOException, PlanningException { new FileFragment[]{frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[7]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); @@ -389,7 +391,7 @@ public final void testStorePlan() throws IOException, PlanningException { ctx.setOutputPath(new Path(workDir, "grouped1")); Expr context = analyzer.parse(CreateTableAsStmts[0]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); @@ -431,7 +433,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio ctx.setOutputPath(new Path(workDir, "grouped2")); Expr context = analyzer.parse(CreateTableAsStmts[1]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.RCFILE); @@ -472,7 +474,7 @@ public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOExcep ctx.setOutputPath(new Path(workDir, "grouped3")); Expr context = analyzer.parse(CreateTableAsStmts[2]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); @@ -483,7 +485,7 @@ public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOExcep public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOException, PlanningException { Expr context = analyzer.parse(CreateTableAsStmts[2]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalRootNode rootNode = (LogicalRootNode) optimizer.optimize(plan); CreateTableNode createTableNode = rootNode.getChild(); Enforcer enforcer = new Enforcer(); @@ -507,7 +509,7 @@ public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOExc public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOException, PlanningException { Expr context = analyzer.parse(CreateTableAsStmts[2]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalRootNode rootNode = (LogicalRootNode) optimizer.optimize(plan); CreateTableNode createTableNode = rootNode.getChild(); Enforcer enforcer = new Enforcer(); @@ -537,7 +539,7 @@ public final void testPartitionedStorePlan() throws IOException, PlanningExcepti id, new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[7]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); int numPartitions = 3; Column key1 = new Column("default.score.deptname", Type.TEXT); @@ -598,7 +600,7 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() id, new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[14]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); LogicalNode rootNode = plan.getRootBlock().getRoot(); int numPartitions = 1; DataChannel dataChannel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(), @@ -653,7 +655,7 @@ public final void testAggregationFunction() throws IOException, PlanningExceptio new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[8]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); // Set all aggregation functions to the first phase mode @@ -684,7 +686,7 @@ public final void testCountFunction() throws IOException, PlanningException { new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[9]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); // Set all aggregation functions to the first phase mode @@ -712,7 +714,7 @@ public final void testGroupByWithNullValue() throws IOException, PlanningExcepti new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[11]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); @@ -737,7 +739,7 @@ public final void testUnionPlan() throws IOException, PlanningException { new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); LogicalRootNode root = (LogicalRootNode) rootNode; UnionNode union = plan.createNode(UnionNode.class); @@ -764,7 +766,7 @@ public final void testEvalExpr() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] { }, workDir); Expr expr = analyzer.parse(QUERIES[12]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); @@ -777,7 +779,7 @@ public final void testEvalExpr() throws IOException, PlanningException { assertTrue(7.0d == tuple.get(1).asFloat8()); expr = analyzer.parse(QUERIES[13]); - plan = planner.createPlan(session, expr); + plan = planner.createPlan(defaultContext, expr); rootNode = optimizer.optimize(plan); phyPlanner = new PhysicalPlannerImpl(conf, sm); @@ -801,7 +803,7 @@ public final void testCreateIndex() throws IOException, PlanningException { LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); Expr context = analyzer.parse(createIndexStmt[0]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); @@ -830,7 +832,7 @@ public final void testDuplicateEliminate() throws IOException, PlanningException new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(duplicateElimination[0]); - LogicalPlan plan = planner.createPlan(session, expr); + LogicalPlan plan = planner.createPlan(defaultContext, expr); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); @@ -864,7 +866,7 @@ public final void testIndexedStoreExec() throws IOException, PlanningException { new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(SORT_QUERY[0]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); SortNode sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT); @@ -953,7 +955,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortEnforcer"); Expr context = analyzer.parse(SORT_QUERY[0]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); optimizer.optimize(plan); LogicalNode rootNode = plan.getRootBlock().getRoot(); @@ -975,7 +977,7 @@ public final void testSortEnforcer() throws IOException, PlanningException { assertTrue(exec instanceof MemSortExec); context = analyzer.parse(SORT_QUERY[0]); - plan = planner.createPlan(session, context); + plan = planner.createPlan(defaultContext, context); optimizer.optimize(plan); rootNode = plan.getRootBlock().getRoot(); @@ -1003,7 +1005,7 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByEnforcer"); Expr context = analyzer.parse(QUERIES[7]); - LogicalPlan plan = planner.createPlan(session, context); + LogicalPlan plan = planner.createPlan(defaultContext, context); optimizer.optimize(plan); LogicalNode rootNode = plan.getRootBlock().getRoot(); @@ -1025,7 +1027,7 @@ public final void testGroupByEnforcer() throws IOException, PlanningException { assertNotNull(PhysicalPlanUtil.findExecutor(exec, HashAggregateExec.class)); context = analyzer.parse(QUERIES[7]); - plan = planner.createPlan(session, context); + plan = planner.createPlan(defaultContext, context); optimizer.optimize(plan); rootNode = plan.getRootBlock().getRoot(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index ef5fe0e1c3..f649dacb28 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -143,7 +143,7 @@ private void testProgress(int sortBufferBytesNum) throws Exception { LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = plan.getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 46b0a8eea2..c5c7b5715a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -62,7 +62,7 @@ public class TestRightOuterHashJoinExec { private LogicalPlanner planner; private AbstractStorageManager sm; private Path testDir; - private static Session session = LocalTajoTestingUtility.createDummySession(); + private QueryContext defaultContext; private TableDesc dep3; private TableDesc job3; @@ -214,6 +214,7 @@ public void setUp() throws Exception { analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); } @After @@ -241,7 +242,7 @@ public final void testRightOuter_HashJoinExec0() throws IOException, PlanningExc LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); @@ -282,7 +283,7 @@ public final void testRightOuter_HashJoinExec1() throws IOException, PlanningExc LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); @@ -323,7 +324,7 @@ public final void testRightOuter_HashJoinExec2() throws IOException, PlanningExc LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[2]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 49a9247b57..bf6b2cb394 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -61,7 +61,8 @@ public class TestRightOuterMergeJoinExec { private LogicalPlanner planner; private AbstractStorageManager sm; private Path testDir; - private static final Session session = LocalTajoTestingUtility.createDummySession(); + private QueryContext defaultContext; + private TableDesc dep3; private TableDesc dep4; @@ -276,10 +277,10 @@ public void setUp() throws Exception { phone3 = CatalogUtil.newTableDesc(PHONE3_NAME, phone3Schema, phone3Meta, phone3Path); catalog.createTable(phone3); - - analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); + + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); } @After @@ -305,7 +306,7 @@ public void tearDown() throws Exception { @Test public final void testRightOuterMergeJoin0() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[0]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -340,7 +341,7 @@ public final void testRightOuterMergeJoin0() throws IOException, PlanningExcepti @Test public final void testRightOuter_MergeJoin1() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[1]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -376,7 +377,7 @@ public final void testRightOuter_MergeJoin1() throws IOException, PlanningExcept @Test public final void testRightOuterMergeJoin2() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[2]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -412,7 +413,7 @@ public final void testRightOuterMergeJoin2() throws IOException, PlanningExcepti @Test public final void testRightOuter_MergeJoin3() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[3]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -449,7 +450,7 @@ public final void testRightOuter_MergeJoin3() throws IOException, PlanningExcept @Test public final void testRightOuter_MergeJoin4() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[4]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); @@ -486,7 +487,7 @@ public final void testRightOuter_MergeJoin4() throws IOException, PlanningExcept @Test public final void testRightOuterMergeJoin5() throws IOException, PlanningException { Expr expr = analyzer.parse(QUERIES[5]); - LogicalNode plan = planner.createPlan(session, expr).getRootBlock().getRoot(); + LogicalNode plan = planner.createPlan(defaultContext, expr).getRootBlock().getRoot(); JoinNode joinNode = PlannerUtil.findTopNode(plan, NodeType.JOIN); Enforcer enforcer = new Enforcer(); enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 48317a75ef..4d4cc3d794 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -115,7 +115,7 @@ public final void testNext() throws IOException, PlanningException { .newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), context); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index ae751aaa3f..64803744bf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -574,7 +574,7 @@ public final void testNumShufflePartition() throws Exception { TajoTestingCluster.createTable("testnumshufflepartition", schema, tableOptions, data.toArray(new String[]{}), 3); try { - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME.varname, "2"); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME.varname, "2"); ResultSet res = executeString( "select col1 \n" + ",count(distinct col2) as cnt1\n" + @@ -628,8 +628,8 @@ public int compare(QueryMasterTask o1, QueryMasterTask o2) { assertEquals(2, partitionIds.size()); executeString("DROP TABLE testnumshufflepartition PURGE").close(); } finally { - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME.varname, - ConfVars.DIST_QUERY_GROUPBY_PARTITION_VOLUME.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME.varname, + ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME.defaultVal); } } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java index bca1e871e7..2625136dcc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java @@ -48,9 +48,9 @@ public class TestJoinBroadcast extends QueryTestCaseBase { public TestJoinBroadcast() throws Exception { super(TajoConstants.DEFAULT_DATABASE_NAME); - testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "true"); + testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "true"); testingCluster.setAllTajoDaemonConfValue( - TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "" + (5 * 1024)); + TajoConf.ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "" + (5 * 1024)); executeDDL("create_lineitem_large_ddl.sql", "lineitem_large"); executeDDL("create_customer_large_ddl.sql", "customer_large"); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 9b507ab69f..9501556e69 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -49,39 +49,39 @@ public class TestJoinQuery extends QueryTestCaseBase { public TestJoinQuery(String joinOption) { super(TajoConstants.DEFAULT_DATABASE_NAME); - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, - ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, - ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, + ConfVars.$TEST_BROADCAST_JOIN_ENABLED.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, + ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.defaultVal); testingCluster.setAllTajoDaemonConfValue( - ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.varname, - ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.defaultVal); + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.varname, - ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, - ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, + ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal); if (joinOption.indexOf("NoBroadcast") >= 0) { - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "false"); - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "-1"); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false"); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, "-1"); } if (joinOption.indexOf("Hash") >= 0) { testingCluster.setAllTajoDaemonConfValue( - ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.varname, String.valueOf(256 * 1048576)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, String.valueOf(256 * 1048576)); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, String.valueOf(256 * 1048576)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, String.valueOf(256 * 1048576)); } if (joinOption.indexOf("Sort") >= 0) { testingCluster.setAllTajoDaemonConfValue( - ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.varname, String.valueOf(1)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, String.valueOf(1)); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, String.valueOf(1)); - testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, String.valueOf(1)); } } @@ -98,19 +98,19 @@ public static Collection generateParameters() { @AfterClass public static void classTearDown() { - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, - ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, - ConfVars.DIST_QUERY_BROADCAST_JOIN_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, + ConfVars.$TEST_BROADCAST_JOIN_ENABLED.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.varname, + ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD.defaultVal); testingCluster.setAllTajoDaemonConfValue( - ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.varname, - ConfVars.EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD.defaultVal); + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.varname, - ConfVars.EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD.defaultVal); - testingCluster.setAllTajoDaemonConfValue(ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, - ConfVars.EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname, + ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname, + ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.defaultVal); } @Test @@ -405,7 +405,7 @@ public final void testLeftOuterJoinWithEmptySubquery1() throws Exception { TajoTestingCluster.createTable("table12", schema, tableOptions, data, 2); try { - testingCluster.setAllTajoDaemonConfValue(ConfVars.$TESTCASE_MIN_TASK_NUM.varname, "2"); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2"); ResultSet res = executeString("select a.id, b.id from table11 a " + "left outer join (" + @@ -423,8 +423,8 @@ public final void testLeftOuterJoinWithEmptySubquery1() throws Exception { assertEquals(expected, resultSetToString(res)); cleanupQuery(res); } finally { - testingCluster.setAllTajoDaemonConfValue(ConfVars.$TESTCASE_MIN_TASK_NUM.varname, - ConfVars.$TESTCASE_MIN_TASK_NUM.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, + ConfVars.$TEST_MIN_TASK_NUM.defaultVal); executeString("DROP TABLE table11 PURGE").close(); executeString("DROP TABLE table12 PURGE").close(); } @@ -447,7 +447,7 @@ public final void testLeftOuterJoinWithEmptySubquery2() throws Exception { TajoTestingCluster.createTable("table12", schema, tableOptions, data, 2); try { - testingCluster.setAllTajoDaemonConfValue(ConfVars.$TESTCASE_MIN_TASK_NUM.varname, "2"); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2"); ResultSet res = executeString("select a.id, b.id from " + "(select table12.id, table12.name, lineitem.l_shipdate " + @@ -461,8 +461,8 @@ public final void testLeftOuterJoinWithEmptySubquery2() throws Exception { assertEquals(expected, resultSetToString(res)); cleanupQuery(res); } finally { - testingCluster.setAllTajoDaemonConfValue(ConfVars.$TESTCASE_MIN_TASK_NUM.varname, - ConfVars.$TESTCASE_MIN_TASK_NUM.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, + ConfVars.$TEST_MIN_TASK_NUM.defaultVal); executeString("DROP TABLE table11 PURGE"); executeString("DROP TABLE table12 PURGE"); } @@ -1077,8 +1077,8 @@ public void testJoinWithDifferentShuffleKey() throws Exception { } TajoTestingCluster.createTable("large_table", schema, tableOptions, data.toArray(new String[]{})); - int originConfValue = conf.getIntVar(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME); - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME.varname, "1"); + int originConfValue = conf.getIntVar(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME.varname, "1"); ResultSet res = executeString( "select count(b.id) " + "from (select id, count(*) as cnt from large_table group by id) a " + @@ -1094,7 +1094,7 @@ public void testJoinWithDifferentShuffleKey() throws Exception { assertEquals(expected, resultSetToString(res)); } finally { - testingCluster.setAllTajoDaemonConfValue(ConfVars.DIST_QUERY_JOIN_PARTITION_VOLUME.varname, "" + originConfValue); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME.varname, "" + originConfValue); cleanupQuery(res); executeString("DROP TABLE large_table PURGE").close(); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index bfc2e21620..dc83c6d421 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -433,7 +433,7 @@ public final void testNowInMultipleTasks() throws Exception { TajoTestingCluster.createTable("table11", schema, tableOptions, data, 2); try { - testingCluster.setAllTajoDaemonConfValue(ConfVars.$TESTCASE_MIN_TASK_NUM.varname, "2"); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2"); ResultSet res = executeString("select concat(substr(to_char(now(),'yyyymmddhh24miss'), 1, 14), 'aaa'), sleep(1) from table11"); @@ -465,8 +465,8 @@ public final void testNowInMultipleTasks() throws Exception { } assertEquals(5, numRecords); } finally { - testingCluster.setAllTajoDaemonConfValue(ConfVars.$TESTCASE_MIN_TASK_NUM.varname, - ConfVars.$TESTCASE_MIN_TASK_NUM.defaultVal); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, + ConfVars.$TEST_MIN_TASK_NUM.defaultVal); executeString("DROP TABLE table11 PURGE"); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java index a775b84740..c7f64061b7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java @@ -181,7 +181,7 @@ public final void testTopkWithJson() throws Exception { @Test public final void testSortNullColumn() throws Exception { try { - testingCluster.setAllTajoDaemonConfValue(ConfVars.$TESTCASE_MIN_TASK_NUM.varname, "2"); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "2"); KeyValueSet tableOptions = new KeyValueSet(); tableOptions.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); tableOptions.set(StorageConstants.CSVFILE_NULL, "\\\\N"); @@ -214,7 +214,7 @@ public final void testSortNullColumn() throws Exception { cleanupQuery(res); } finally { - testingCluster.setAllTajoDaemonConfValue(ConfVars.$TESTCASE_MIN_TASK_NUM.varname, "0"); + testingCluster.setAllTajoDaemonConfValue(ConfVars.$TEST_MIN_TASK_NUM.varname, "0"); executeString("DROP TABLE nullsort PURGE;").close(); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 6a1a3cafca..6bbf63bca5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -61,7 +61,7 @@ public static void setUp() throws Exception { util.startCatalogCluster(); conf = util.getConfiguration(); - conf.set(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "false"); + conf.set(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false"); catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:!234/warehouse"); @@ -107,7 +107,7 @@ public void testNextBlock() throws Exception { "join supplier on s_nationkey = n_nationkey " + "join partsupp on s_suppkey = ps_suppkey " + "join part on p_partkey = ps_partkey and p_type like '%BRASS' and p_size = 15"); - LogicalPlan logicalPlan = logicalPlanner.createPlan(LocalTajoTestingUtility.createDummySession(), context); + LogicalPlan logicalPlan = logicalPlanner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context); optimizer.optimize(logicalPlan); QueryContext queryContext = new QueryContext(conf); MasterPlan plan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), queryContext, logicalPlan); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java index 2d482ebe18..b6ac55128e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java @@ -112,7 +112,7 @@ public static void tearDown() { private MasterPlan buildPlan(String sql) throws PlanningException, IOException { Expr expr = sqlAnalyzer.parse(sql); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(util.getConfiguration()), expr); optimizer.optimize(plan); QueryContext context = new QueryContext(util.getConfiguration()); MasterPlan masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), context, plan); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java index e340953fa7..b4be00bec1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java @@ -44,7 +44,7 @@ public TestQueryUnitStatusUpdate() { @BeforeClass public static void setUp() throws Exception { - conf.set(TajoConf.ConfVars.DIST_QUERY_BROADCAST_JOIN_AUTO.varname, "false"); + conf.set(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false"); } @Test diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index 875527f87a..5f8efe7116 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -148,7 +148,7 @@ public void testGet() throws Exception { new FileFragment[] {frags[0]}, testDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(SORT_QUERY[0]); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); @@ -272,7 +272,7 @@ public void testGetFromDescendingOrder() throws Exception { new FileFragment[] {frags[0]}, testDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(SORT_QUERY[1]); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummySession(), expr); + LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm); From 9a1239e607fdb3cb56cda45e2fc9483901a27b37 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 4 Aug 2014 03:13:32 +0900 Subject: [PATCH 12/13] Added unit tests and refactored many classes to use SessionVars. --- .../cli/DefaultTajoCliOutputFormatter.java | 16 +- .../java/org/apache/tajo/cli/HelpCommand.java | 102 +++++--- .../java/org/apache/tajo/cli/SetCommand.java | 80 +++++- .../java/org/apache/tajo/cli/TajoCli.java | 149 +++++++---- .../tajo/cli/TajoCliOutputFormatter.java | 4 +- .../org/apache/tajo/cli/TajoShellCommand.java | 8 + .../main/java/org/apache/tajo/ConfigKey.java | 28 +- .../java/org/apache/tajo/OverridableConf.java | 239 ++++++++++++++++++ .../java/org/apache/tajo/SessionVars.java | 84 +++--- .../java/org/apache/tajo/conf/TajoConf.java | 5 +- .../org/apache/tajo/util/KeyValueSet.java | 5 + .../engine/planner/PhysicalPlannerImpl.java | 37 ++- .../tajo/engine/query/QueryContext.java | 148 ++--------- .../tajo/master/TajoMasterClientService.java | 8 +- .../master/querymaster/Repartitioner.java | 2 +- .../cli/TestDefaultCliOutputFormatter.java | 49 +++- .../java/org/apache/tajo/cli/TestTajoCli.java | 163 ++++++------ .../planner/physical/TestHashJoinExec.java | 7 +- .../TestTajoCli/testHelpSessionVars.result | 36 +++ ...tSelectResultWithNullTrueDeprecated.result | 9 + .../testStopWhenErrorDeprecated.result | 6 + 21 files changed, 820 insertions(+), 365 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/OverridableConf.java create mode 100644 tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result create mode 100644 tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrueDeprecated.result create mode 100644 tajo-core/src/test/resources/results/TestTajoCli/testStopWhenErrorDeprecated.result diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java index a88c197a01..543ac8cb33 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java @@ -20,11 +20,10 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.tajo.QueryId; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.client.QueryStatus; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.util.FileUtil; import java.io.InputStream; @@ -33,20 +32,17 @@ import java.sql.ResultSetMetaData; public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { - private TajoConf tajoConf; private int printPauseRecords; private boolean printPause; private boolean printErrorTrace; private String nullChar; @Override - public void init(TajoConf tajoConf) { - this.tajoConf = tajoConf; - - this.printPause = tajoConf.getBoolVar(TajoConf.ConfVars.$CLI_PRINT_PAUSE); - this.printPauseRecords = tajoConf.getIntVar(TajoConf.ConfVars.$CLI_PRINT_PAUSE_NUM_RECORDS); - this.printErrorTrace = tajoConf.getBoolVar(TajoConf.ConfVars.$CLI_PRINT_ERROR_TRACE); - this.nullChar = tajoConf.getVar(ConfVars.$CLI_NULL_CHAR); + public void init(TajoCli.TajoCliContext context) { + this.printPause = context.getBool(SessionVars.CLI_PAGING_ENABLED); + this.printPauseRecords = context.getInt(SessionVars.CLI_PAGE_ROWS); + this.printErrorTrace = context.getBool(SessionVars.CLI_DISPLAY_ERROR_TRACE); + this.nullChar = context.get(SessionVars.CLI_NULL_CHAR); } @Override diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java index 8d908ec7d4..d686ff0c6d 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java @@ -18,6 +18,7 @@ package org.apache.tajo.cli; +import com.google.common.collect.Maps; import org.apache.tajo.util.VersionInfo; import java.io.IOException; @@ -26,6 +27,7 @@ import java.net.MalformedURLException; import java.net.ProtocolException; import java.net.URL; +import java.util.Map; public class HelpCommand extends TajoShellCommand { private String targetDocVersion = ""; @@ -39,54 +41,70 @@ public String getCommand() { return "\\?"; } + @Override + public String [] getAliases() { + return new String [] {"\\help"}; + } + @Override public void invoke(String[] cmd) throws Exception { if(targetDocVersion.equalsIgnoreCase("")) { targetDocVersion = getDocumentationVersion(); } - PrintWriter sout = context.getOutput(); - sout.println(); - - sout.println("General"); - sout.println(" \\copyright show Apache License 2.0"); - sout.println(" \\version show Tajo version"); - sout.println(" \\? show help"); - sout.println(" \\q quit tsql"); - sout.println(); - sout.println(); - - sout.println("Informational"); - sout.println(" \\l list databases"); - sout.println(" \\c show current database"); - sout.println(" \\c [DBNAME] connect to new database"); - sout.println(" \\d list tables"); - sout.println(" \\d [TBNAME] describe table"); - sout.println(" \\df list functions"); - sout.println(" \\df NAME describe function"); - sout.println(); - sout.println(); - - sout.println("Tool"); - sout.println(" \\! execute a linux shell command"); - sout.println(" \\dfs execute a dfs command"); - sout.println(" \\admin execute tajo admin command"); - sout.println(); - sout.println(); - - sout.println("Variables"); - sout.println(" \\set [[NAME] [VALUE] set session variable or list session variables"); - sout.println(" \\unset NAME unset session variable"); - sout.println(); - sout.println(); - - sout.println("Documentations"); - sout.println(" tsql guide http://tajo.apache.org/docs/"+ targetDocVersion +"/cli.html"); - sout.println(" Query language http://tajo.apache.org/docs/"+ targetDocVersion +"/sql_language.html"); - sout.println(" Functions http://tajo.apache.org/docs/"+ targetDocVersion +"/functions.html"); - sout.println(" Backup & restore http://tajo.apache.org/docs/"+ targetDocVersion +"/backup_and_restore.html"); - sout.println(" Configuration http://tajo.apache.org/docs/"+ targetDocVersion +"/configuration.html"); - sout.println(); + if (cmd.length == 1) { + PrintWriter sout = context.getOutput(); + sout.println(); + + sout.println("General"); + sout.println(" \\copyright show Apache License 2.0"); + sout.println(" \\version show Tajo version"); + sout.println(" \\? show help"); + sout.println(" \\? [COMMAND] show help of a given command"); + sout.println(" \\help alias of \\?"); + sout.println(" \\q quit tsql"); + sout.println(); + sout.println(); + + sout.println("Informational"); + sout.println(" \\l list databases"); + sout.println(" \\c show current database"); + sout.println(" \\c [DBNAME] connect to new database"); + sout.println(" \\d list tables"); + sout.println(" \\d [TBNAME] describe table"); + sout.println(" \\df list functions"); + sout.println(" \\df NAME describe function"); + sout.println(); + sout.println(); + + sout.println("Tool"); + sout.println(" \\! execute a linux shell command"); + sout.println(" \\dfs execute a dfs command"); + sout.println(" \\admin execute tajo admin command"); + sout.println(); + sout.println(); + + sout.println("Variables"); + sout.println(" \\set [[NAME] [VALUE] set session variable or list session variables"); + sout.println(" \\unset NAME unset session variable"); + sout.println(); + sout.println(); + + sout.println("Documentations"); + sout.println(" tsql guide http://tajo.apache.org/docs/" + targetDocVersion + "/cli.html"); + sout.println(" Query language http://tajo.apache.org/docs/" + targetDocVersion + "/sql_language.html"); + sout.println(" Functions http://tajo.apache.org/docs/" + targetDocVersion + "/functions.html"); + sout.println(" Backup & restore http://tajo.apache.org/docs/" + targetDocVersion + "/backup_and_restore.html"); + sout.println(" Configuration http://tajo.apache.org/docs/" + targetDocVersion + "/configuration.html"); + sout.println(); + } else if (cmd.length == 2) { + String slashCommand = "\\" + cmd[1]; + if (context.getCommands().containsKey(slashCommand)) { + context.getCommands().get(slashCommand).printHelp(); + } else { + context.getOutput().println("Command not found: " + cmd[1]); + } + } } private String getDocumentationVersion() { diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java index c694507534..7e777a28bb 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java @@ -18,11 +18,17 @@ package org.apache.tajo.cli; +import com.google.protobuf.ServiceException; +import com.sun.org.apache.xpath.internal.operations.Bool; +import org.apache.tajo.SessionVars; import org.apache.tajo.util.StringUtils; +import java.io.PrintWriter; import java.util.HashMap; import java.util.Map; +import static org.apache.tajo.SessionVars.VariableMode; + public class SetCommand extends TajoShellCommand { public SetCommand(TajoCli.TajoCliContext context) { @@ -34,17 +40,46 @@ public String getCommand() { return "\\set"; } + private void showAllSessionVars() throws ServiceException { + for (Map.Entry entry: client.getAllSessionVariables().entrySet()) { + context.getOutput().println(StringUtils.quote(entry.getKey()) + "=" + StringUtils.quote(entry.getValue())); + } + } + + private void updateSessionVariable(String key, String val) throws ServiceException { + Map variables = new HashMap(); + variables.put(key, val); + client.updateSessionVariables(variables); + } + + void set(String key, String val) throws ServiceException { + SessionVars sessionVar = null; + + if (SessionVars.exists(key)) { // if the variable is one of the session variables + sessionVar = SessionVars.get(key); + + // is it cli-side variable? + if (sessionVar.getMode() == VariableMode.CLI_SIDE_VAR) { + context.setCliSideVar(key, val); + } else { + updateSessionVariable(key, val); + } + + if (SessionVars.isDeprecated(key)) { + context.getOutput().println("Warning: deprecated to directly use config key in TajoConf.ConfVars. " + + "Please execute '\\help set'."); + } + } else { + updateSessionVariable(key, val); + } + } + @Override public void invoke(String[] cmd) throws Exception { if (cmd.length == 1) { - for (Map.Entry entry: client.getAllSessionVariables().entrySet()) { - context.getOutput().println(StringUtils.quote(entry.getKey()) + "=" + StringUtils.quote(entry.getValue())); - } + showAllSessionVars(); } else if (cmd.length == 3) { - Map variables = new HashMap(); - variables.put(cmd[1], cmd[2]); - client.updateSessionVariables(variables); - context.setVariable(cmd[1], cmd[2]); + set(cmd[1], cmd[2]); } else { context.getOutput().println("usage: \\set [[NAME] VALUE]"); } @@ -59,4 +94,35 @@ public String getUsage() { public String getDescription() { return "set session variable or shows all session variables"; } + + @Override + public void printHelp() { + context.getOutput().println("Available Session Variables:\n"); + for (SessionVars var : SessionVars.values()) { + + if (var.getMode() == VariableMode.DEFAULT || + var.getMode() == VariableMode.CLI_SIDE_VAR || + var.getMode() == VariableMode.FROM_SHELL_ENV) { + + context.getOutput().println("\\set " + var.keyname() + " " + getDisplayType(var.getVarType()) + " - " + var + .getDescription()); + } + } + } + + public static String getDisplayType(Class clazz) { + if (clazz == String.class) { + return "[text value]"; + } else if (clazz == Integer.class) { + return "[int value]"; + } else if (clazz == Long.class) { + return "[long value]"; + } else if (clazz == Float.class) { + return "[real value]"; + } else if (clazz == Boolean.class) { + return "[true or false]"; + } else { + return clazz.getSimpleName(); + } + } } diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java index 54eaefafce..b8651b1b4f 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java @@ -18,11 +18,12 @@ package org.apache.tajo.cli; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.protobuf.ServiceException; import jline.console.ConsoleReader; import org.apache.commons.cli.*; -import org.apache.tajo.QueryId; -import org.apache.tajo.QueryIdFactory; +import org.apache.tajo.*; import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; @@ -62,7 +63,7 @@ public class TajoCli { // Current States private String currentDatabase; - private TajoCliOutputFormatter outputFormatter; + private TajoCliOutputFormatter displayFormatter; private boolean wasError = false; @@ -98,7 +99,10 @@ public class TajoCli { options.addOption("help", "help", false, "help"); } - public class TajoCliContext { + public class TajoCliContext extends OverridableConf { + public TajoCliContext(TajoConf conf) { + super(conf, ConfigKey.ConfigType.SESSION); + } public TajoClient getTajoClient() { return client; @@ -120,18 +124,50 @@ public TajoConf getConf() { return conf; } - public void setVariable(String key, String value) { - conf.set(key, value); - try { - initFormatter(); - } catch (Exception e) { - System.err.println(ERROR_PREFIX + e.getMessage()); + @VisibleForTesting + public String getCliSideVar(String key) { + if (SessionVars.exists(key)) { + ConfigKey configKey = SessionVars.get(key); + return get(configKey); + } else { + return get(key); + } + } + + public void setCliSideVar(String key, String value) { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(value); + + boolean shouldReloadFormatter = false; + + if (SessionVars.exists(key)) { + SessionVars configKey = SessionVars.get(key); + put(configKey, value); + shouldReloadFormatter = configKey.getMode() == SessionVars.VariableMode.CLI_SIDE_VAR; + } else { + set(key, value); + + // It is hard to recognize it is a client side variable. So, we always reload formatter. + shouldReloadFormatter = true; } + + if (shouldReloadFormatter) { + try { + initFormatter(); + } catch (Exception e) { + System.err.println(ERROR_PREFIX + e.getMessage()); + } + } + } + + public Map getCommands() { + return commands; } } public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) throws Exception { this.conf = new TajoConf(c); + context = new TajoCliContext(conf); this.sin = in; this.reader = new ConsoleReader(sin, out); this.reader.setExpandEvents(false); @@ -160,13 +196,7 @@ public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) thr } if (cmd.getOptionValues("conf") != null) { - for (String eachParam: cmd.getOptionValues("conf")) { - String[] tokens = eachParam.split("="); - if (tokens.length != 2) { - continue; - } - conf.set(tokens[0], tokens[1]); - } + processConfVarCommand(cmd.getOptionValues("conf")); } // if there is no "-h" option, @@ -195,19 +225,22 @@ public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) thr client = new TajoClient(conf, baseDatabase); } - context = new TajoCliContext(); context.setCurrentDatabase(client.getCurrentDatabase()); initHistory(); initCommands(); + if (cmd.getOptionValues("conf") != null) { + processSessionVarCommand(cmd.getOptionValues("conf")); + } + if (cmd.hasOption("c")) { - outputFormatter.setScirptMode(); + displayFormatter.setScirptMode(); int exitCode = executeScript(cmd.getOptionValue("c")); sout.flush(); System.exit(exitCode); } if (cmd.hasOption("f")) { - outputFormatter.setScirptMode(); + displayFormatter.setScirptMode(); cmd.getOptionValues(""); File sqlFile = new File(cmd.getOptionValue("f")); if (sqlFile.exists()) { @@ -225,13 +258,38 @@ public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) thr addShutdownHook(); } + private void processConfVarCommand(String[] confCommands) throws ServiceException { + for (String eachParam: confCommands) { + String[] tokens = eachParam.split("="); + if (tokens.length != 2) { + continue; + } + + if (!SessionVars.exists(tokens[0])) { + conf.set(tokens[0], tokens[1]); + } + } + } + + private void processSessionVarCommand(String[] confCommands) throws ServiceException { + for (String eachParam: confCommands) { + String[] tokens = eachParam.split("="); + if (tokens.length != 2) { + continue; + } + + if (SessionVars.exists(tokens[0])) { + ((SetCommand)commands.get("\\set")).set(tokens[0], tokens[1]); + } + } + } + private void initFormatter() throws Exception { - Class formatterClass = conf.getClass(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS.varname, - DefaultTajoCliOutputFormatter.class); - if (outputFormatter == null || !outputFormatter.getClass().equals(formatterClass)) { - outputFormatter = (TajoCliOutputFormatter)formatterClass.newInstance(); + Class formatterClass = context.getClass(SessionVars.CLI_FORMATTER_CLASS); + if (displayFormatter == null || !displayFormatter.getClass().equals(formatterClass)) { + displayFormatter = (TajoCliOutputFormatter)formatterClass.newInstance(); } - outputFormatter.init(conf); + displayFormatter.init(context); } public TajoCliContext getContext() { @@ -279,6 +337,9 @@ private void initCommands() { throw new RuntimeException(e.getMessage()); } commands.put(cmd.getCommand(), cmd); + for (String alias : cmd.getAliases()) { + commands.put(alias, cmd); + } } } @@ -331,7 +392,7 @@ public int runShell() throws Exception { exitCode = executeParsedResults(parsedResults); currentPrompt = updatePrompt(parser.getState()); - if (exitCode != 0 && context.getConf().getBoolVar(ConfVars.$CLI_ERROR_STOP)) { + if (exitCode != 0 && context.getBool(SessionVars.ON_ERROR_STOP)) { return exitCode; } } @@ -371,18 +432,18 @@ public int executeMetaCommand(String line) throws Exception { try { invoked.invoke(arguments); } catch (IllegalArgumentException ige) { - outputFormatter.printErrorMessage(sout, ige); + displayFormatter.printErrorMessage(sout, ige); wasError = true; return -1; } catch (Exception e) { - outputFormatter.printErrorMessage(sout, e); + displayFormatter.printErrorMessage(sout, e); wasError = true; return -1; } finally { context.getOutput().flush(); } - if (wasError && context.getConf().getBoolVar(ConfVars.$CLI_ERROR_STOP)) { + if (wasError && context.getBool(SessionVars.ON_ERROR_STOP)) { break; } } @@ -394,7 +455,7 @@ private void executeJsonQuery(String json) throws ServiceException { long startTime = System.currentTimeMillis(); ClientProtos.SubmitQueryResponse response = client.executeQueryWithJson(json); if (response == null) { - outputFormatter.printErrorMessage(sout, "response is null"); + displayFormatter.printErrorMessage(sout, "response is null"); wasError = true; } else if (response.getResultCode() == ClientProtos.ResultCode.OK) { if (response.getIsForwarded()) { @@ -402,7 +463,7 @@ private void executeJsonQuery(String json) throws ServiceException { waitForQueryCompleted(queryId); } else { if (!response.hasTableDesc() && !response.hasResultSet()) { - outputFormatter.printMessage(sout, "OK"); + displayFormatter.printMessage(sout, "OK"); wasError = true; } else { localQueryCompleted(response, startTime); @@ -410,7 +471,7 @@ private void executeJsonQuery(String json) throws ServiceException { } } else { if (response.hasErrorMessage()) { - outputFormatter.printErrorMessage(sout, response.getErrorMessage()); + displayFormatter.printErrorMessage(sout, response.getErrorMessage()); wasError = true; } } @@ -420,7 +481,7 @@ private int executeQuery(String statement) throws ServiceException { long startTime = System.currentTimeMillis(); ClientProtos.SubmitQueryResponse response = client.executeQuery(statement); if (response == null) { - outputFormatter.printErrorMessage(sout, "response is null"); + displayFormatter.printErrorMessage(sout, "response is null"); wasError = true; } else if (response.getResultCode() == ClientProtos.ResultCode.OK) { if (response.getIsForwarded()) { @@ -428,14 +489,14 @@ private int executeQuery(String statement) throws ServiceException { waitForQueryCompleted(queryId); } else { if (!response.hasTableDesc() && !response.hasResultSet()) { - outputFormatter.printMessage(sout, "OK"); + displayFormatter.printMessage(sout, "OK"); } else { localQueryCompleted(response, startTime); } } } else { if (response.hasErrorMessage()) { - outputFormatter.printErrorMessage(sout, response.getErrorMessage()); + displayFormatter.printErrorMessage(sout, response.getErrorMessage()); wasError = true; } } @@ -453,13 +514,13 @@ private void localQueryCompleted(ClientProtos.SubmitQueryResponse response, long // non-forwarded INSERT INTO query does not have any query id. // In this case, it just returns succeeded query information without printing the query results. if (response.getMaxRowNum() < 0 && queryId.equals(QueryIdFactory.NULL_QUERY_ID)) { - outputFormatter.printResult(sout, sin, desc, responseTime, res); + displayFormatter.printResult(sout, sin, desc, responseTime, res); } else { res = TajoClient.createResultSet(client, response); - outputFormatter.printResult(sout, sin, desc, responseTime, res); + displayFormatter.printResult(sout, sin, desc, responseTime, res); } } catch (Throwable t) { - outputFormatter.printErrorMessage(sout, t); + displayFormatter.printErrorMessage(sout, t); wasError = true; } finally { if (res != null) { @@ -494,7 +555,7 @@ private void waitForQueryCompleted(QueryId queryId) { } if (status.getState() == QueryState.QUERY_RUNNING || status.getState() == QueryState.QUERY_SUCCEEDED) { - outputFormatter.printProgress(sout, status); + displayFormatter.printProgress(sout, status); } if (status.getState() != QueryState.QUERY_RUNNING && @@ -508,10 +569,10 @@ private void waitForQueryCompleted(QueryId queryId) { } if (status.getState() == QueryState.QUERY_ERROR || status.getState() == QueryState.QUERY_FAILED) { - outputFormatter.printErrorMessage(sout, status); + displayFormatter.printErrorMessage(sout, status); wasError = true; } else if (status.getState() == QueryState.QUERY_KILLED) { - outputFormatter.printKilledMessage(sout, queryId); + displayFormatter.printKilledMessage(sout, queryId); wasError = true; } else { if (status.getState() == QueryState.QUERY_SUCCEEDED) { @@ -520,15 +581,15 @@ private void waitForQueryCompleted(QueryId queryId) { if (status.hasResult()) { res = TajoClient.createResultSet(client, queryId, response); TableDesc desc = new TableDesc(response.getTableDesc()); - outputFormatter.printResult(sout, sin, desc, responseTime, res); + displayFormatter.printResult(sout, sin, desc, responseTime, res); } else { TableDesc desc = new TableDesc(response.getTableDesc()); - outputFormatter.printResult(sout, sin, desc, responseTime, res); + displayFormatter.printResult(sout, sin, desc, responseTime, res); } } } } catch (Throwable t) { - outputFormatter.printErrorMessage(sout, t); + displayFormatter.printErrorMessage(sout, t); wasError = true; } finally { if (res != null) { diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java index 0e9166941d..60cafb05a8 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java @@ -30,9 +30,9 @@ public interface TajoCliOutputFormatter { /** * Initialize formatter - * @param tajoConf + * @param context */ - public void init(TajoConf tajoConf); + public void init(TajoCli.TajoCliContext context); /** * print query result to console diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java index bc4fea91c3..39f5377b07 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java @@ -23,9 +23,17 @@ public abstract class TajoShellCommand { public abstract String getCommand(); + public String [] getAliases() { + return new String[] {}; + } public abstract void invoke(String [] command) throws Exception; public abstract String getUsage(); public abstract String getDescription(); + public void printHelp() { + context.getOutput().print(getCommand()); + context.getOutput().print(" - "); + context.getOutput().println(getDescription()); + } protected TajoCli.TajoCliContext context; protected TajoClient client; diff --git a/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java index 12b1542d35..b9d51ec76d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java +++ b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java @@ -19,13 +19,31 @@ package org.apache.tajo; public interface ConfigKey { - public static final String SESSION_PREFIX = "$"; - public static final String QUERY_CONF_PREFIX = "%"; + + // Client can set or change variables of this mode. + public static final int DEFAULT_MODE = 0; + // This is similar to DEFAULT mode. In addition, it tries to get values from shell env. variables. + public static final int FROM_SHELL_ENV_MODE = 1; + // only TajoMaster is able to set and change variables of this mode. + public static final int SERVER_SIDE_VAR_MODE = 2; + // This type variable will be used only in cli side. + public static final int CLI_SIDE_VAR_MODE = 3; public static enum ConfigType { - SYSTEM, - SESSION, - QUERY + SYSTEM(""), + SESSION("$"), + QUERY("@"), + CLI("+"); + + private String prefix; + + ConfigType(String prefix) { + this.prefix = prefix; + } + + public String getPrefix() { + return prefix; + } } public String keyname(); diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java new file mode 100644 index 0000000000..19607189bc --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import com.google.common.base.Preconditions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.util.KeyValueSet; + +import java.lang.ref.WeakReference; +import java.util.Map; +import java.util.WeakHashMap; + +import static org.apache.tajo.ConfigKey.ConfigType; +import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; + +/** + * QueryContext provides a consolidated config system for a query instant. + * + * In Tajo, there are three configurable layers: + *
    + *
  • + *
      System Config - it comes from Hadoop's Configuration class. by tajo-site, catalog-site, + * catalog-default and TajoConf.
    + *
      Session variables - they are instantly configured by users. + * Each client session has it own set of session variables.
    + *
      Query config - it is internally used for meta information of a query instance.
    + *
    • + *
+ * + * System configs and session variables can set the same config in the same time. System configs are usually used to set + * default configs, and session variables is user-specified configs. So, session variables can override system configs. + * + * QueryContent provides a query with a uniform way to access various configs without considering their priorities. + */ +public class OverridableConf extends KeyValueSet { + private static final Log LOG = LogFactory.getLog(OverridableConf.class); + private ConfigType [] configTypes; + + private static final Map>>> + CACHE_CLASSES = new WeakHashMap>>>(); + + private TajoConf conf; + + public OverridableConf(final TajoConf conf, ConfigType...configTypes) { + this.conf = conf; + this.configTypes = configTypes; + } + + public OverridableConf(final TajoConf conf, KeyValueSetProto proto, ConfigType...configTypes) { + super(proto); + this.conf = conf; + this.configTypes = configTypes; + } + + public void setConf(TajoConf conf) { + this.conf = conf; + } + + public TajoConf getConf() { + return conf; + } + + public void setBool(ConfigKey key, boolean val) { + setBool(key.keyname(), val); + } + + public boolean getBool(ConfigKey key, Boolean defaultVal) { + assertRegisteredEnum(key); + + switch (key.type()) { + case QUERY: + return getBool(key.keyname()); + case SESSION: + return getBool(key.keyname(), conf.getBoolVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getBoolVar((TajoConf.ConfVars) key); + default: + return getBool(key.keyname(), defaultVal); + } + } + + public boolean getBool(ConfigKey key) { + return getBool(key, null); + } + + public int getInt(ConfigKey key, Integer defaultVal) { + assertRegisteredEnum(key); + + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { + return getInt(key.keyname()); + } else { + switch (key.type()) { + case SESSION: + return getInt(key.keyname(), conf.getIntVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getIntVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } + } + } + + public int getInt(ConfigKey key) { + return getInt(key, null); + } + + public long getLong(ConfigKey key, Long defaultVal) { + assertRegisteredEnum(key); + + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { + return getLong(key.keyname()); + } else { + switch (key.type()) { + case SESSION: + return getLong(key.keyname(), conf.getLongVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getLongVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } + } + } + + public long getLong(ConfigKey key) { + return getLong(key, null); + } + + public float getFloat(ConfigKey key, Float defaultVal) { + assertRegisteredEnum(key); + + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { + return getFloat(key.keyname()); + } else { + switch (key.type()) { + case SESSION: + return getFloat(key.keyname(), conf.getFloatVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getFloatVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } + } + } + + public float getFloat(ConfigKey key) { + return getLong(key, null); + } + + public void put(ConfigKey key, String val) { + set(key.keyname(), val); + } + + private void assertRegisteredEnum(ConfigKey key) { + boolean registered = false; + + for (ConfigType c : configTypes) { + registered = key.type() == c; + } + + registered |= key.type() == ConfigType.SESSION || key.type() != ConfigType.SYSTEM; + + Preconditions.checkArgument(registered, key.keyname() + " (" + key.type() + ") is not allowed in " + + getClass().getSimpleName()); + } + + public String get(ConfigKey key, String defaultVal) { + assertRegisteredEnum(key); + + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { + return get(key.keyname(), defaultVal); + } else { + switch (key.type()) { + case SESSION: + return get(key.keyname(), conf.getVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } + } + } + + public String get(ConfigKey key) { + return get(key, null); + } + + public Class getClass(ConfigKey key) { +// if (containsKey(key)) { + String className = getTrimmed(key); + try { + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } +// } else { +// throw new IllegalArgumentException("No such a config key: " + key); +// } + } + + public String getTrimmed(ConfigKey key) { + String value = get(key); + + if (null == value) { + return null; + } else { + return value.trim(); + } + } + + public boolean containsKey(ConfigKey key) { + return containsKey(key.keyname()); + } + + public boolean equalKey(ConfigKey key, String another) { + if (containsKey(key)) { + return get(key).equals(another); + } else { + return false; + } + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index 4c7589f604..b281dca7d8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -53,23 +53,23 @@ public enum SessionVars implements ConfigKey { //------------------------------------------------------------------------------- // Client -------------------------------------------------------- - SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "", DEFAULT), + SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "session expiry time (secs)", DEFAULT), // Command line interface and its behavior -------------------------------- - // TODO - they should be replaced by '\pset' meta command - DISPLAY_WIDTH(ConfVars.$CLI_MAX_COLUMN, "", DEFAULT), - DISPLAY_FORMATTER_CLASS(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, "", DEFAULT), - DISPLAY_NULL_CHAR(ConfVars.$CLI_NULL_CHAR, "", DEFAULT), + CLI_COLUMNS(ConfVars.$CLI_MAX_COLUMN, "Sets the width for the wrapped format", CLI_SIDE_VAR), + CLI_FORMATTER_CLASS(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, "Sets the output format class to display results", + CLI_SIDE_VAR), + CLI_NULL_CHAR(ConfVars.$CLI_NULL_CHAR, "Sets the string to be printed in place of a null value.", CLI_SIDE_VAR), - CLI_PRINT_PAUSE_NUM_RECORDS(ConfVars.$CLI_PRINT_PAUSE_NUM_RECORDS, "", DEFAULT), - CLI_PRINT_PAUSE(ConfVars.$CLI_PRINT_PAUSE, "", DEFAULT), - CLI_PRINT_ERROR_TRACE(ConfVars.$CLI_PRINT_ERROR_TRACE, "", DEFAULT), + CLI_PAGE_ROWS(ConfVars.$CLI_PRINT_PAUSE_NUM_RECORDS, "Sets the number of rows for paging", CLI_SIDE_VAR), + CLI_PAGING_ENABLED(ConfVars.$CLI_PRINT_PAUSE, "Enable paging of result display", CLI_SIDE_VAR), + CLI_DISPLAY_ERROR_TRACE(ConfVars.$CLI_PRINT_ERROR_TRACE, "Enable display of error trace", CLI_SIDE_VAR), - ON_ERROR_STOP(ConfVars.$CLI_ERROR_STOP, "tsql will exist if an error occurs.", DEFAULT), + ON_ERROR_STOP(ConfVars.$CLI_ERROR_STOP, "tsql will exist if an error occurs.", CLI_SIDE_VAR), // Timezone & Date ---------------------------------------------------------- - TZ(ConfVars.$TIMEZONE, "Timezone", FROM_SHELL_ENV), - DATE_ORDER(ConfVars.$DATE_ORDER, "YMD", FROM_SHELL_ENV), + TZ(ConfVars.$TIMEZONE, "Sets timezone", FROM_SHELL_ENV), + DATE_ORDER(ConfVars.$DATE_ORDER, "date order (default is YMD)", FROM_SHELL_ENV), // Locales and Character set ------------------------------------------------ // TODO - they are reserved variables, and we should support them. @@ -86,24 +86,30 @@ public enum SessionVars implements ConfigKey { // Query and Optimization --------------------------------------------------- // for distributed query strategies - BROADCAST_TABLE_SIZE_LIMIT(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, "", DEFAULT), + BROADCAST_TABLE_SIZE_LIMIT(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, "limited size (bytes) of broadcast table", + DEFAULT), - JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size", DEFAULT), - SORT_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_SORT_TASK_VOLUME, "sort task input size", DEFAULT), - GROUPBY_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_GROUPBY_TASK_VOLUME, "group by task input size", DEFAULT), + JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size (mb) ", DEFAULT), + SORT_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_SORT_TASK_VOLUME, "sort task input size (mb)", DEFAULT), + GROUPBY_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_GROUPBY_TASK_VOLUME, "group by task input size (mb)", DEFAULT), - JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join", DEFAULT), - GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort", DEFAULT), - TABLE_PARTITION_WRITE_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME, // TODO - rename - "shuffle output size for partition table write", DEFAULT), + JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join (mb)", DEFAULT), + GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort (mb)", DEFAULT), + TABLE_PARTITION_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME, + "shuffle output size for partition table write (mb)", DEFAULT), // for physical Executors - EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort", DEFAULT), - HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "hash join size limit", DEFAULT), - HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD, "hash groupby size limit", DEFAULT), - MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size", DEFAULT), - - NULL_CHAR(ConfVars.$CSVFILE_NULL, "null char of text files", DEFAULT), + EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort (mb)", DEFAULT), + HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash join (mb)", DEFAULT), + INNER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD, + "limited size for hash inner join (mb)", DEFAULT), + OUTER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash outer join (mb)", + DEFAULT), + HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD, "limited size for hash groupby (mb)", + DEFAULT), + MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size (mb). 0 means infinite.", DEFAULT), + + NULL_CHAR(ConfVars.$CSVFILE_NULL, "null char of text file output", DEFAULT), // Behavior Control --------------------------------------------------------- ARITHABORT(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT, @@ -113,17 +119,19 @@ public enum SessionVars implements ConfigKey { // Only for Unit Testing //------------------------------------------------------------------------------- DEBUG_ENABLED(ConfVars.$DEBUG_ENABLED, "(debug only) debug mode enabled", DEFAULT), - TEST_BROADCAST_JOIN_ENABLED(ConfVars.$TEST_BROADCAST_JOIN_ENABLED, "(test only) broadcast enabled", DEFAULT), - TEST_JOIN_OPT_ENABLED(ConfVars.$TEST_JOIN_OPT_ENABLED, "(test only) join optimization enabled", DEFAULT), - TEST_FILTER_PUSHDOWN_ENABLED(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED, "filter push down enabled", DEFAULT), - TEST_MIN_TASK_NUM(ConfVars.$TEST_MIN_TASK_NUM, "(test only) min task num", DEFAULT), + TEST_BROADCAST_JOIN_ENABLED(ConfVars.$TEST_BROADCAST_JOIN_ENABLED, "(test only) broadcast enabled", TEST_VAR), + TEST_JOIN_OPT_ENABLED(ConfVars.$TEST_JOIN_OPT_ENABLED, "(test only) join optimization enabled", TEST_VAR), + TEST_FILTER_PUSHDOWN_ENABLED(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED, "filter push down enabled", TEST_VAR), + TEST_MIN_TASK_NUM(ConfVars.$TEST_MIN_TASK_NUM, "(test only) min task num", TEST_VAR), ; private static Map SESSION_VARS = Maps.newHashMap(); + private static Map DEPRECATED_SESSION_VARS = Maps.newHashMap(); static { for (SessionVars var : SessionVars.values()) { SESSION_VARS.put(var.keyname(), var); + DEPRECATED_SESSION_VARS.put(var.getConfVars().keyname(), var); } } @@ -135,6 +143,8 @@ public static enum VariableMode { DEFAULT, // Client can set or change variables of this mode.. FROM_SHELL_ENV, // This is similar to DEFAULT mode. In addition, it tries to get values from shell env. variables. SERVER_SIDE_VAR, // only TajoMaster is able to set and change variables of this mode. + CLI_SIDE_VAR, // This type variable is used in CLI. + TEST_VAR // Only used for unit tests } SessionVars(ConfVars key, String description, VariableMode mode) { @@ -144,7 +154,7 @@ public static enum VariableMode { } public String keyname() { - return SESSION_PREFIX + name(); + return name(); } public ConfigType type() { @@ -167,13 +177,19 @@ public VariableMode getMode() { return mode; } - public static boolean exists(String name) { - return SESSION_VARS.containsKey(name.toUpperCase()); + public static boolean exists(String keyname) { + return SESSION_VARS.containsKey(keyname) || DEPRECATED_SESSION_VARS.containsKey(keyname); + } + + public static boolean isDeprecated(String keyname) { + return DEPRECATED_SESSION_VARS.containsKey(keyname); } public static SessionVars get(String keyname) { - if (exists(keyname)) { - return SESSION_VARS.get(keyname.toUpperCase()); + if (SESSION_VARS.containsKey(keyname)) { + return SESSION_VARS.get(keyname); + } else if (DEPRECATED_SESSION_VARS.containsKey(keyname)) { + return DEPRECATED_SESSION_VARS.get(keyname); } else { return null; } diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 794865b433..67ecbe82ae 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -311,6 +311,10 @@ public static enum ConfVars implements ConfigKey { $EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L), $EXECUTOR_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.common.in-memory-hash-threshold-bytes", (long)256 * 1048576), + $EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.inner.in-memory-hash-threshold-bytes", + (long)256 * 1048576), + $EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.outer.in-memory-hash-threshold-bytes", + (long)256 * 1048576), $EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-bytes", (long)256 * 1048576), $MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite @@ -526,7 +530,6 @@ public void setBoolVar(ConfVars var, boolean val) { } public static String getVar(Configuration conf, ConfVars var) { - assert (var.valClass == String.class); return conf.get(var.varname, var.defaultVal); } diff --git a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java index 4cba082406..0c3db6ddd3 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/KeyValueSet.java @@ -19,7 +19,9 @@ package org.apache.tajo.util; import com.google.common.base.Objects; +import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.json.CommonGsonHelper; import org.apache.tajo.json.GsonObject; @@ -89,6 +91,9 @@ public boolean containsKey(String key) { } public void set(String key, String val) { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(val); + this.keyVals.put(key, val); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index d065d18556..9f533e272e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -38,6 +38,7 @@ import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.physical.*; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InternalException; import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer; @@ -255,7 +256,16 @@ public boolean checkIfInMemoryInnerJoinIsPossible(TaskAttemptContext context, Lo throws IOException { String [] lineage = PlannerUtil.getRelationLineage(node); long volume = estimateSizeRecursive(context, lineage); - boolean inMemoryInnerJoinFlag = volume <= context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT); + boolean inMemoryInnerJoinFlag = false; + + QueryContext queryContext = context.getQueryContext(); + + if (queryContext.containsKey(SessionVars.INNER_HASH_JOIN_SIZE_LIMIT)) { + inMemoryInnerJoinFlag = volume <= context.getQueryContext().getLong(SessionVars.INNER_HASH_JOIN_SIZE_LIMIT); + } else { + inMemoryInnerJoinFlag = volume <= context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT); + } + LOG.info(String.format("[%s] the volume of %s relations (%s) is %s and is %sfit to main maemory.", context.getTaskId().toString(), (left ? "Left" : "Right"), @@ -467,8 +477,17 @@ private PhysicalExec createBestLeftOuterJoinPlan(TaskAttemptContext context, Joi PhysicalExec leftExec, PhysicalExec rightExec) throws IOException { String [] rightLineage = PlannerUtil.getRelationLineage(plan.getRightChild()); long rightTableVolume = estimateSizeRecursive(context, rightLineage); + boolean hashJoin; + + QueryContext queryContext = context.getQueryContext(); + + if (queryContext.containsKey(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT)) { + hashJoin = rightTableVolume < queryContext.getLong(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT); + } else { + hashJoin = rightTableVolume < queryContext.getLong(SessionVars.HASH_JOIN_SIZE_LIMIT); + } - if (rightTableVolume < context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT)) { + if (hashJoin) { // we can implement left outer join using hash join, using the right operand as the build relation LOG.info("Left Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, leftExec, rightExec); @@ -485,8 +504,18 @@ private PhysicalExec createBestRightJoinPlan(TaskAttemptContext context, JoinNod //if the left operand is small enough => implement it as a left outer hash join with exchanged operators (note: // blocking, but merge join is blocking as well) String [] outerLineage4 = PlannerUtil.getRelationLineage(plan.getLeftChild()); - long outerSize = estimateSizeRecursive(context, outerLineage4); - if (outerSize < context.getQueryContext().getLong(SessionVars.HASH_JOIN_SIZE_LIMIT)){ + long leftTableVolume = estimateSizeRecursive(context, outerLineage4); + boolean hashJoin; + + QueryContext queryContext = context.getQueryContext(); + + if (queryContext.containsKey(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT)) { + hashJoin = leftTableVolume < queryContext.getLong(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT); + } else { + hashJoin = leftTableVolume < queryContext.getLong(SessionVars.HASH_JOIN_SIZE_LIMIT); + } + + if (hashJoin){ LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Hash Join]."); return new HashLeftOuterJoinExec(context, plan, rightExec, leftExec); } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 52d5df08d7..79f85347d8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -22,13 +22,16 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.ConfigKey; +import org.apache.tajo.OverridableConf; import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.master.session.Session; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.util.KeyValueSet; +import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; /** @@ -50,17 +53,16 @@ * * QueryContent provides a query with a uniform way to access various configs without considering their priorities. */ -public class QueryContext extends KeyValueSet { - +public class QueryContext extends OverridableConf { public static enum QueryVars implements ConfigKey { COMMAND_TYPE, - STAGING_DIR(), - OUTPUT_TABLE_NAME(), - OUTPUT_TABLE_PATH(), - OUTPUT_PARTITIONS(), - OUTPUT_OVERWRITE(), - OUTPUT_AS_DIRECTORY(), - OUTPUT_PER_FILE_SIZE(), + STAGING_DIR, + OUTPUT_TABLE_NAME, + OUTPUT_TABLE_PATH, + OUTPUT_PARTITIONS, + OUTPUT_OVERWRITE, + OUTPUT_AS_DIRECTORY, + OUTPUT_PER_FILE_SIZE, ; QueryVars() { @@ -68,7 +70,7 @@ public static enum QueryVars implements ConfigKey { @Override public String keyname() { - return QUERY_CONF_PREFIX + name(); + return name().toLowerCase(); } @Override @@ -77,131 +79,17 @@ public ConfigType type() { } } - private static final Log LOG = LogFactory.getLog(QueryContext.class); - - private TajoConf conf; - - public QueryContext(final TajoConf conf) { - this.conf = conf; + public QueryContext(TajoConf conf) { + super(conf, ConfigKey.ConfigType.QUERY); } - public QueryContext(final TajoConf conf, Session session) { - this.conf = conf; + public QueryContext(TajoConf conf, Session session) { + super(conf); putAll(session.getAllVariables()); } - public QueryContext(final TajoConf conf, KeyValueSetProto proto) { - super(proto); - this.conf = conf; - } - - public void setConf(TajoConf conf) { - this.conf = conf; - } - - public TajoConf getConf() { - return conf; - } - - public void setBool(ConfigKey key, boolean val) { - setBool(key.keyname(), val); - } - - public boolean getBool(ConfigKey key, Boolean defaultVal) { - switch (key.type()) { - case QUERY: - return getBool(key.keyname()); - case SESSION: - return getBool(key.keyname(), conf.getBoolVar(((SessionVars) key).getConfVars())); - case SYSTEM: - return conf.getBoolVar((TajoConf.ConfVars) key); - default: - return getBool(key.keyname(), defaultVal); - } - } - - public boolean getBool(ConfigKey key) { - return getBool(key, null); - } - - public int getInt(ConfigKey key, Integer defaultVal) { - switch (key.type()) { - case QUERY: - return getInt(key.keyname()); - case SESSION: - return getInt(key.keyname(), conf.getIntVar(((SessionVars) key).getConfVars())); - case SYSTEM: - return conf.getIntVar((TajoConf.ConfVars) key); - default: - return getInt(key.keyname(), defaultVal); - } - } - - public int getInt(ConfigKey key) { - return getInt(key, null); - } - - public long getLong(ConfigKey key, Long defaultVal) { - switch (key.type()) { - case QUERY: - return getLong(key.keyname()); - case SESSION: - return getLong(key.keyname(), conf.getLongVar(((SessionVars) key).getConfVars())); - case SYSTEM: - return conf.getLongVar((TajoConf.ConfVars) key); - default: - return getLong(key.keyname(), defaultVal); - } - } - - public long getLong(ConfigKey key) { - return getLong(key, null); - } - - public float getFloat(ConfigKey key, Float defaultVal) { - switch (key.type()) { - case QUERY: - return getFloat(key.keyname()); - case SESSION: - return getFloat(key.keyname(), conf.getFloatVar(((SessionVars) key).getConfVars())); - case SYSTEM: - return conf.getFloatVar((TajoConf.ConfVars) key); - default: - return getFloat(key.keyname(), defaultVal); - } - } - - public float getFloat(ConfigKey key) { - return getLong(key, null); - } - - public void put(ConfigKey key, String val) { - set(key.keyname(), val); - } - - public static String getSessionKey(String key) { - return ConfigKey.SESSION_PREFIX + key; - } - - public static String getQueryKey(String key) { - return ConfigKey.QUERY_CONF_PREFIX + key; - } - - public String get(ConfigKey key, String defaultVal) { - switch (key.type()) { - case QUERY: - return get(key.keyname()); - case SESSION: - return get(key.keyname(), conf.getVar(((SessionVars) key).getConfVars())); - case SYSTEM: - return conf.getVar((TajoConf.ConfVars) key); - default: - return get(key.keyname(), defaultVal); - } - } - - public String get(ConfigKey key) { - return get(key, null); + public QueryContext(TajoConf conf, KeyValueSetProto proto) { + super(conf, proto, ConfigKey.ConfigType.QUERY); } //----------------------------------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 93efe1a79a..7d80a88f1f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -27,10 +27,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; -import org.apache.tajo.QueryId; -import org.apache.tajo.QueryIdFactory; -import org.apache.tajo.TajoIdProtos; -import org.apache.tajo.TajoProtos; +import org.apache.tajo.*; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.NoSuchDatabaseException; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -197,7 +194,8 @@ public StringProto getSessionVariable(RpcController controller, SessionedStringP } @Override - public BoolProto existSessionVariable(RpcController controller, SessionedStringProto request) throws ServiceException { + public BoolProto existSessionVariable(RpcController controller, SessionedStringProto request) + throws ServiceException { try { String value = context.getSessionManager().getVariable(request.getSessionId().getId(), request.getValue()); if (value != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index 899db07d5f..e79fe0d207 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -772,7 +772,7 @@ public static void scheduleScatteredHashShuffleFetches(TaskSchedulerContext sche String tableName) { int i = 0; long splitVolume = StorageUnit.MB * - subQuery.getMasterPlan().getContext().getLong(SessionVars.TABLE_PARTITION_WRITE_PER_SHUFFLE_SIZE); + subQuery.getMasterPlan().getContext().getLong(SessionVars.TABLE_PARTITION_PER_SHUFFLE_SIZE); long sumNumBytes = 0L; Map> fetches = new HashMap>(); diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestDefaultCliOutputFormatter.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestDefaultCliOutputFormatter.java index e13eeef6de..eb8b8c69db 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/TestDefaultCliOutputFormatter.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestDefaultCliOutputFormatter.java @@ -18,6 +18,9 @@ package org.apache.tajo.cli; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.TpchTestBase; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; @@ -27,18 +30,56 @@ import org.apache.tajo.datum.TextDatum; import org.apache.tajo.jdbc.MetaDataTuple; import org.apache.tajo.jdbc.TajoMetaDataResultSet; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import java.io.ByteArrayOutputStream; import java.io.PrintWriter; import java.io.StringWriter; +import java.net.URL; import java.sql.ResultSet; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static org.mockito.Mockito.*; + import static org.junit.Assert.assertEquals; public class TestDefaultCliOutputFormatter { + protected static final TpchTestBase testBase; + protected static final TajoTestingCluster cluster; + + /** the base path of result directories */ + protected static final Path resultBasePath; + static { + testBase = TpchTestBase.getInstance(); + cluster = testBase.getTestingCluster(); + URL resultBaseURL = ClassLoader.getSystemResource("results"); + resultBasePath = new Path(resultBaseURL.toString()); + } + + private TajoConf conf; + private TajoCli tajoCli; + private TajoCli.TajoCliContext cliContext; + + @Before + public void setUp() throws Exception { + conf = cluster.getConfiguration(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + tajoCli = new TajoCli(conf, new String[]{}, System.in, out); + cliContext = tajoCli.getContext(); + } + + @After + public void tearDown() { + if (tajoCli != null) { + tajoCli.close(); + } + } + + @Test public void testParseErrorMessage() { String message = "java.sql.SQLException: ERROR: no such a table: table1"; @@ -65,9 +106,10 @@ public void testParseErrorMessage() { @Test public void testPrintResultInsertStatement() throws Exception { - TajoConf tajoConf = new TajoConf(); + + DefaultTajoCliOutputFormatter outputFormatter = new DefaultTajoCliOutputFormatter(); - outputFormatter.init(tajoConf); + outputFormatter.init(cliContext); float responseTime = 10.1f; long numBytes = 102; @@ -89,9 +131,8 @@ public void testPrintResultInsertStatement() throws Exception { @Test public void testPrintResultSelectStatement() throws Exception { - TajoConf tajoConf = new TajoConf(); DefaultTajoCliOutputFormatter outputFormatter = new DefaultTajoCliOutputFormatter(); - outputFormatter.init(tajoConf); + outputFormatter.init(cliContext); float responseTime = 10.1f; long numBytes = 102; diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java index 0ed0c581b3..e8efddbcd3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestTajoCli.java @@ -23,14 +23,16 @@ import org.apache.commons.cli.PosixParser; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.ConfigKey; +import org.apache.tajo.SessionVars; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.TpchTestBase; import org.apache.tajo.client.QueryStatus; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.FileUtil; import org.junit.After; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; @@ -40,9 +42,7 @@ import java.io.PrintWriter; import java.net.URL; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class TestTajoCli { protected static final TpchTestBase testBase; @@ -59,6 +59,7 @@ public class TestTajoCli { private TajoCli tajoCli; private Path currentResultPath; + private ByteArrayOutputStream out; @Rule public TestName name = new TestName(); @@ -68,6 +69,12 @@ public TestTajoCli() { currentResultPath = new Path(resultBasePath, className); } + @Before + public void setUp() throws Exception { + out = new ByteArrayOutputStream(); + tajoCli = new TajoCli(cluster.getConfiguration(), new String[]{}, System.in, out); + } + @After public void tearDown() { if (tajoCli != null) { @@ -75,6 +82,40 @@ public void tearDown() { } } + private static void setVar(TajoCli cli, ConfigKey key, String val) throws Exception { + cli.executeMetaCommand("\\set " + key.keyname() +" " + val); + } + + private static void assertSessionVar(TajoCli cli, String key, String expectedVal) { + assertEquals(cli.getContext().getCliSideVar(key), expectedVal); + } + + private void assertOutputResult(String actual) throws Exception { + assertOutputResult(name.getMethodName() + ".result", actual); + } + + private void assertOutputResult(String expectedResultFile, String actual) throws Exception { + assertOutputResult(expectedResultFile, actual, null, null); + } + + private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues) + throws Exception { + FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration()); + Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile); + assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile)); + + String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri())); + + if (paramKeys != null) { + for (int i = 0; i < paramKeys.length; i++) { + if (i < paramValues.length) { + expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]); + } + } + } + assertEquals(expectedResult.trim(), actual.trim()); + } + @Test public void testParseParam() throws Exception { String[] args = new String[]{"-f", "test.sql", "--param", "test1=10", "--param", "test2=20"}; @@ -111,10 +152,9 @@ public void testParseConf() throws Exception { assertEquals("tajo.executor.join.inner.in-memory-table-num=256", confValues[1]); TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - - tajoCli = new TajoCli(tajoConf, args, System.in, System.out); - assertEquals("false", tajoCli.getContext().getConf().get("tajo.cli.print.pause")); - assertEquals("256", tajoCli.getContext().getConf().get("tajo.executor.join.inner.in-memory-table-num")); + TajoCli testCli = new TajoCli(tajoConf, args, System.in, System.out); + assertEquals("false", testCli.getContext().get(SessionVars.CLI_PAGING_ENABLED)); + assertEquals("256", testCli.getContext().getConf().get("tajo.executor.join.inner.in-memory-table-num")); } @Test @@ -130,10 +170,7 @@ public void testReplaceParam() throws Exception { @Test public void testLocalQueryWithoutFrom() throws Exception { String sql = "select 'abc', '123'; select substr('123456', 1,3);"; - TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - tajoConf.setVar(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); tajoCli.executeScript(sql); String consoleResult = new String(out.toByteArray()); @@ -150,11 +187,7 @@ public void testConnectDatabase() throws Exception { databaseName = "TEST_CONNECTION_DATABASE"; } String sql = "create database \"" + databaseName + "\";"; - TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - tajoConf.setVar(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); tajoCli.executeScript(sql); tajoCli.executeMetaCommand("\\c " + databaseName); @@ -178,11 +211,7 @@ public void testDescTable() throws Exception { String sql = "create table \"" + tableName + "\" (col1 int4, col2 int4);"; - TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - tajoConf.setVar(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); tajoCli.executeScript(sql); tajoCli.executeMetaCommand("\\d " + tableName); @@ -210,79 +239,51 @@ public void testSelectResultWithNullFalse() throws Exception { " c_custkey,\n" + " orders.o_orderkey;\n"; - TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - tajoConf.setVar(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - - tajoConf.setVar(ConfVars.$CLI_NULL_CHAR, ""); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); tajoCli.executeScript(sql); String consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); } - @Test - public void testSelectResultWithNullTrue() throws Exception { + private void verifySelectResultWithNullTrue() throws Exception { String sql = "select\n" + - " c_custkey,\n" + - " orders.o_orderkey,\n" + - " orders.o_orderstatus \n" + - "from\n" + - " orders full outer join customer on c_custkey = o_orderkey\n" + - "order by\n" + - " c_custkey,\n" + - " orders.o_orderkey;\n"; + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; - TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - tajoConf.setVar(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - tajoConf.setVar(ConfVars.$CLI_NULL_CHAR, "testnull"); + setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); + assertSessionVar(tajoCli, SessionVars.CLI_NULL_CHAR.keyname(), "testnull"); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); tajoCli.executeScript(sql); String consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); } - private void assertOutputResult(String actual) throws Exception { - assertOutputResult(name.getMethodName() + ".result", actual); - } - - private void assertOutputResult(String expectedResultFile, String actual) throws Exception { - assertOutputResult(expectedResultFile, actual, null, null); + @Test + public void testSelectResultWithNullTrueDeprecated() throws Exception { + setVar(tajoCli, TajoConf.ConfVars.$CLI_NULL_CHAR, "testnull"); + verifySelectResultWithNullTrue(); } - private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues) - throws Exception { - FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration()); - Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile); - assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile)); - - String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri())); - - if (paramKeys != null) { - for (int i = 0; i < paramKeys.length; i++) { - if (i < paramValues.length) { - expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]); - } - } - } - assertEquals(expectedResult, actual); + @Test + public void testSelectResultWithNullTrue() throws Exception { + setVar(tajoCli, SessionVars.CLI_NULL_CHAR, "testnull"); + verifySelectResultWithNullTrue(); } - @Test - public void testStopWhenError() throws Exception { - TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - tajoConf.setVar(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); + private void verifyStopWhenError() throws Exception { + setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); - tajoCli.executeMetaCommand("\\set tajo.cli.error.stop true"); + assertSessionVar(tajoCli, SessionVars.ON_ERROR_STOP.keyname(), "true"); tajoCli.executeScript("select count(*) from lineitem; " + "select count(*) from lineitem2; " + @@ -292,6 +293,24 @@ public void testStopWhenError() throws Exception { assertOutputResult(consoleResult); } + @Test + public void testStopWhenErrorDeprecated() throws Exception { + tajoCli.executeMetaCommand("\\set tajo.cli.error.stop true"); + verifyStopWhenError(); + } + + @Test + public void testStopWhenError() throws Exception { + tajoCli.executeMetaCommand("\\set ON_ERROR_STOP true"); + verifyStopWhenError(); + } + + @Test + public void testHelpSessionVars() throws Exception { + tajoCli.executeMetaCommand("\\help set"); + assertOutputResult(new String(out.toByteArray())); + } + public static class TajoCliOutputTestFormatter extends DefaultTajoCliOutputFormatter { @Override protected String getResponseTimeReadable(float responseTime) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index 5bc05c2173..d6f02a94e3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.LocalTajoTestingUtility; +import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.algebra.Expr; @@ -201,10 +202,8 @@ public final void testCheckIfInMemoryInnerJoinIsPossible() throws IOException, P LocalTajoTestingUtility.newQueryUnitAttemptId(), merged, workDir); ctx.setEnforcer(enforcer); - TajoConf localConf = new TajoConf(conf); - localConf.setLongVar(TajoConf.ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, 100l); - ctx.getQueryContext().setConf(localConf); - PhysicalPlannerImpl phyPlanner = new PhysicalPlannerImpl(localConf, sm); + ctx.getQueryContext().setLong(SessionVars.HASH_JOIN_SIZE_LIMIT.keyname(), 100l); + PhysicalPlannerImpl phyPlanner = new PhysicalPlannerImpl(conf, sm); PhysicalExec exec = phyPlanner.createPlan(ctx, plan); ProjectionExec proj = (ProjectionExec) exec; diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result new file mode 100644 index 0000000000..c294407abd --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result @@ -0,0 +1,36 @@ +Available Session Variables: + +\set SESSION_EXPIRY_TIME [int value] - session expiry time (secs) +\set CLI_COLUMNS [int value] - Sets the width for the wrapped format +\set CLI_FORMATTER_CLASS [text value] - Sets the output format class to display results +\set CLI_NULL_CHAR [text value] - Sets the string to be printed in place of a null value. +\set CLI_PAGE_ROWS [int value] - Sets the number of rows for paging +\set CLI_PAGING_ENABLED [true or false] - Enable paging of result display +\set CLI_DISPLAY_ERROR_TRACE [true or false] - Enable display of error trace +\set ON_ERROR_STOP [true or false] - tsql will exist if an error occurs. +\set TZ [text value] - Sets timezone +\set DATE_ORDER [text value] - date order (default is YMD) +\set LANG [text value] - Language +\set LC_ALL [text value] - String sort order +\set LC_COLLATE [text value] - String sort order +\set LC_CTYPE [text value] - Character classification (What is a letter? Its upper-case equivalent?) +\set LC_MESSAGES [text value] - Language of messages +\set LC_MONETARY [text value] - Formatting of currency amounts +\set LC_NUMERIC [text value] - Formatting of numbers +\set LC_TIME [text value] - Formatting of dates and times +\set BROADCAST_TABLE_SIZE_LIMIT [long value] - limited size (bytes) of broadcast table +\set JOIN_TASK_INPUT_SIZE [int value] - join task input size (mb) +\set SORT_TASK_INPUT_SIZE [int value] - sort task input size (mb) +\set GROUPBY_TASK_INPUT_SIZE [int value] - group by task input size (mb) +\set JOIN_PER_SHUFFLE_SIZE [int value] - shuffle output size for join (mb) +\set GROUPBY_PER_SHUFFLE_SIZE [int value] - shuffle output size for sort (mb) +\set TABLE_PARTITION_PER_SHUFFLE_SIZE [int value] - shuffle output size for partition table write (mb) +\set EXTSORT_BUFFER_SIZE [long value] - sort buffer size for external sort (mb) +\set HASH_JOIN_SIZE_LIMIT [long value] - limited size for hash join (mb) +\set INNER_HASH_JOIN_SIZE_LIMIT [long value] - limited size for hash inner join (mb) +\set OUTER_HASH_JOIN_SIZE_LIMIT [long value] - limited size for hash outer join (mb) +\set HASH_GROUPBY_SIZE_LIMIT [long value] - limited size for hash groupby (mb) +\set MAX_OUTPUT_FILE_SIZE [int value] - Maximum per-output file size (mb). 0 means infinite. +\set NULL_CHAR [text value] - null char of text file output +\set ARITHABORT [true or false] - If true, a running query will be terminated when an overflow or divide-by-zero occurs. +\set DEBUG_ENABLED [true or false] - (debug only) debug mode enabled \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrueDeprecated.result b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrueDeprecated.result new file mode 100644 index 0000000000..36ea548165 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testSelectResultWithNullTrueDeprecated.result @@ -0,0 +1,9 @@ +Warning: deprecated to directly use config key in TajoConf.ConfVars. Please execute '\help set'. +c_custkey, o_orderkey, o_orderstatus +------------------------------- +1, 1, O +2, 2, O +3, 3, F +4, testnull, testnull +5, testnull, testnull +(5 rows, , 30 B selected) \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testStopWhenErrorDeprecated.result b/tajo-core/src/test/resources/results/TestTajoCli/testStopWhenErrorDeprecated.result new file mode 100644 index 0000000000..5c49139d0b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testStopWhenErrorDeprecated.result @@ -0,0 +1,6 @@ +Warning: deprecated to directly use config key in TajoConf.ConfVars. Please execute '\help set'. +?count +------------------------------- +5 +(1 rows, , 2 B selected) +ERROR: relation "default.lineitem2" does not exist From 439fa0667c42b2b4d704b5921fc6da3f683381f5 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 7 Aug 2014 14:36:16 +0900 Subject: [PATCH 13/13] Add deprecated handler and querydetail.jsp. --- .../java/org/apache/tajo/cli/SetCommand.java | 2 +- .../java/org/apache/tajo/OverridableConf.java | 72 +++++++++---------- .../java/org/apache/tajo/SessionVars.java | 18 ++++- .../tajo/engine/query/QueryContext.java | 18 +---- .../tajo/master/querymaster/QueryMaster.java | 11 +-- .../tajo/master/querymaster/SubQuery.java | 7 +- .../apache/tajo/master/session/Session.java | 6 +- .../resources/webapps/worker/querydetail.jsp | 10 +++ 8 files changed, 76 insertions(+), 68 deletions(-) diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java index 7e777a28bb..47e0e78a97 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java @@ -97,7 +97,7 @@ public String getDescription() { @Override public void printHelp() { - context.getOutput().println("Available Session Variables:\n"); + context.getOutput().println("\nAvailable Session Variables:\n"); for (SessionVars var : SessionVars.values()) { if (var.getMode() == VariableMode.DEFAULT || diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java index 19607189bc..c2bb1f94c4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java @@ -32,31 +32,29 @@ import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; /** - * QueryContext provides a consolidated config system for a query instant. + * OverridableConf provides a consolidated config system. Tajo basically uses TajoConf, which is a extended class of + * Hadoop's Configuration system, However, TajoConf is only used for sharing static system configs, such as binding + * address of master and workers, system directories, other system parameters. + * + * For modifiable or instant configs, we use OverridableConf, which is a set of key-value pairs. + * OverridableConf provides more strong-typed way to set configs and its behavior is more clear than Configuration + * system. + * + * By default, OverridableConf recognizes following config types. * - * In Tajo, there are three configurable layers: *
    - *
  • - *
      System Config - it comes from Hadoop's Configuration class. by tajo-site, catalog-site, - * catalog-default and TajoConf.
    - *
      Session variables - they are instantly configured by users. - * Each client session has it own set of session variables.
    - *
      Query config - it is internally used for meta information of a query instance.
    - *
    • + *
  • System Config - it comes from Hadoop's Configuration class. by tajo-site, catalog-site, + * catalog-default and TajoConf.
    • + *
  • Session variables - they are instantly configured by users. + * Each client session has it own set of session variables.
    • *
* * System configs and session variables can set the same config in the same time. System configs are usually used to set * default configs, and session variables is user-specified configs. So, session variables can override system configs. - * - * QueryContent provides a query with a uniform way to access various configs without considering their priorities. */ public class OverridableConf extends KeyValueSet { private static final Log LOG = LogFactory.getLog(OverridableConf.class); private ConfigType [] configTypes; - - private static final Map>>> - CACHE_CLASSES = new WeakHashMap>>>(); - private TajoConf conf; public OverridableConf(final TajoConf conf, ConfigType...configTypes) { @@ -85,15 +83,19 @@ public void setBool(ConfigKey key, boolean val) { public boolean getBool(ConfigKey key, Boolean defaultVal) { assertRegisteredEnum(key); - switch (key.type()) { - case QUERY: - return getBool(key.keyname()); - case SESSION: - return getBool(key.keyname(), conf.getBoolVar(((SessionVars) key).getConfVars())); - case SYSTEM: - return conf.getBoolVar((TajoConf.ConfVars) key); - default: + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { return getBool(key.keyname(), defaultVal); + } else { + switch (key.type()) { + case QUERY: + return getBool(key.keyname()); + case SESSION: + return getBool(key.keyname(), conf.getBoolVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getBoolVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } } } @@ -105,7 +107,7 @@ public int getInt(ConfigKey key, Integer defaultVal) { assertRegisteredEnum(key); if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { - return getInt(key.keyname()); + return getInt(key.keyname(), defaultVal); } else { switch (key.type()) { case SESSION: @@ -126,7 +128,7 @@ public long getLong(ConfigKey key, Long defaultVal) { assertRegisteredEnum(key); if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { - return getLong(key.keyname()); + return getLong(key.keyname(), defaultVal); } else { switch (key.type()) { case SESSION: @@ -147,7 +149,7 @@ public float getFloat(ConfigKey key, Float defaultVal) { assertRegisteredEnum(key); if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { - return getFloat(key.keyname()); + return getFloat(key.keyname(), defaultVal); } else { switch (key.type()) { case SESSION: @@ -203,16 +205,14 @@ public String get(ConfigKey key) { } public Class getClass(ConfigKey key) { -// if (containsKey(key)) { - String className = getTrimmed(key); - try { - return Class.forName(className); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } -// } else { -// throw new IllegalArgumentException("No such a config key: " + key); -// } + assertRegisteredEnum(key); + + String className = getTrimmed(key); + try { + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } } public String getTrimmed(ConfigKey key) { diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index b281dca7d8..5503aaa37a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -125,8 +125,8 @@ public enum SessionVars implements ConfigKey { TEST_MIN_TASK_NUM(ConfVars.$TEST_MIN_TASK_NUM, "(test only) min task num", TEST_VAR), ; - private static Map SESSION_VARS = Maps.newHashMap(); - private static Map DEPRECATED_SESSION_VARS = Maps.newHashMap(); + public static Map SESSION_VARS = Maps.newHashMap(); + public static Map DEPRECATED_SESSION_VARS = Maps.newHashMap(); static { for (SessionVars var : SessionVars.values()) { @@ -185,6 +185,10 @@ public static boolean isDeprecated(String keyname) { return DEPRECATED_SESSION_VARS.containsKey(keyname); } + public static boolean isPublic(SessionVars var) { + return var.getMode() != SERVER_SIDE_VAR; + } + public static SessionVars get(String keyname) { if (SESSION_VARS.containsKey(keyname)) { return SESSION_VARS.get(keyname); @@ -194,4 +198,14 @@ public static SessionVars get(String keyname) { return null; } } + + /** + * rename deprecated name to current name if the name is deprecated. + * + * @param keyname session variable name + * @return The current session variable name + */ + public static String handleDeprecatedName(String keyname) { + return SessionVars.exists(keyname) ? SessionVars.get(keyname).keyname() : keyname; + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 79f85347d8..41d3c360a6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -35,23 +35,7 @@ import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; /** - * QueryContext provides a consolidated config system for a query instant. - * - * In Tajo, there are three configurable layers: - *
    - *
  • - *
      System Config - it comes from Hadoop's Configuration class. by tajo-site, catalog-site, - * catalog-default and TajoConf.
    - *
      Session variables - they are instantly configured by users. - * Each client session has it own set of session variables.
    - *
      Query config - it is internally used for meta information of a query instance.
    - *
    • - *
- * - * System configs and session variables can set the same config in the same time. System configs are usually used to set - * default configs, and session variables is user-specified configs. So, session variables can override system configs. - * - * QueryContent provides a query with a uniform way to access various configs without considering their priorities. + * QueryContent is a overridable config, and it provides a set of various configs for a query instance. */ public class QueryContext extends OverridableConf { public static enum QueryVars implements ConfigKey { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java index 25af82f56a..aed69b2651 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java @@ -29,10 +29,12 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.tajo.QueryId; +import org.apache.tajo.SessionVars; import org.apache.tajo.TajoIdProtos; import org.apache.tajo.TajoProtos; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.GlobalPlanner; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.TajoMasterProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.master.TajoAsyncDispatcher; @@ -83,6 +85,8 @@ public class QueryMaster extends CompositeService implements EventHandler { private QueryMasterContext queryMasterContext; + private QueryContext queryContext; + private QueryHeartbeatThread queryHeartbeatThread; private FinishedQueryMasterTaskCleanThread finishedQueryMasterTaskCleanThread; @@ -362,12 +366,9 @@ public void stopQuery(QueryId queryId) { try { queryMasterTask.stop(); - //if (!systemConf.get(CommonTestingUtil.TAJO_TEST, "FALSE").equalsIgnoreCase("TRUE") - // && !workerContext.isYarnContainerMode()) { - if (!getContext().getConf().getBoolVar(TajoConf.ConfVars.TAJO_DEBUG)) { + if (!queryContext.getBool(SessionVars.DEBUG_ENABLED)) { cleanup(queryId); } - //} } catch (Exception e) { LOG.error(e.getMessage(), e); } @@ -408,6 +409,8 @@ public void handle(QueryStartEvent event) { queryMasterTask.start(); } + queryContext = event.getQueryContext(); + synchronized(queryMasterTasks) { queryMasterTasks.put(event.getQueryId(), queryMasterTask); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index ac65d93751..b6fe9da733 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -32,10 +32,7 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.state.*; import org.apache.hadoop.yarn.util.Records; -import org.apache.tajo.ExecutionBlockId; -import org.apache.tajo.QueryIdFactory; -import org.apache.tajo.QueryUnitId; -import org.apache.tajo.TajoIdProtos; +import org.apache.tajo.*; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; @@ -1110,7 +1107,7 @@ private void cleanup() { stopScheduler(); releaseContainers(); - if (!getContext().getConf().getBoolVar(TajoConf.ConfVars.TAJO_DEBUG)) { + if (!getContext().getQueryContext().getBool(SessionVars.DEBUG_ENABLED)) { List childs = getMasterPlan().getChilds(getId()); List ebIds = Lists.newArrayList(); for (ExecutionBlock executionBlock : childs){ diff --git a/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java b/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java index cf4abcbeb5..cdf552d367 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/session/Session.java @@ -76,14 +76,14 @@ public long getLastAccessTime() { public void setVariable(String name, String value) { synchronized (sessionVariables) { - sessionVariables.put(name, value); + sessionVariables.put(SessionVars.handleDeprecatedName(name), value); } } public String getVariable(String name) throws NoSuchSessionVariableException { synchronized (sessionVariables) { if (sessionVariables.containsKey(name)) { - return sessionVariables.get(name); + return sessionVariables.get(SessionVars.handleDeprecatedName(name)); } else { throw new NoSuchSessionVariableException(name); } @@ -92,7 +92,7 @@ public String getVariable(String name) throws NoSuchSessionVariableException { public void removeVariable(String name) { synchronized (sessionVariables) { - sessionVariables.remove(name); + sessionVariables.remove(SessionVars.handleDeprecatedName(name)); } } diff --git a/tajo-core/src/main/resources/webapps/worker/querydetail.jsp b/tajo-core/src/main/resources/webapps/worker/querydetail.jsp index c0bee9b64b..3c40e99605 100644 --- a/tajo-core/src/main/resources/webapps/worker/querydetail.jsp +++ b/tajo-core/src/main/resources/webapps/worker/querydetail.jsp @@ -29,6 +29,8 @@ <%@ page import="org.apache.tajo.worker.TajoWorker" %> <%@ page import="java.text.SimpleDateFormat" %> <%@ page import="java.util.List" %> +<%@ page import="java.util.Map" %> +<%@ page import="org.apache.tajo.SessionVars" %> <% QueryId queryId = TajoIdUtils.parseQueryId(request.getParameter("queryId")); @@ -93,6 +95,14 @@ for(SubQuery eachSubQuery: subQueries) { %>

+

Applied Session Variables

+ + <%for(Map.Entry entry: query.getPlan().getContext().getAllKeyValus().entrySet()) { + if (SessionVars.exists(entry.getKey()) && SessionVars.isPublic(SessionVars.get(entry.getKey()))) { + %> <% + } + } %> +
<%=entry.getKey()%><%=entry.getValue()%>

Logical Plan

<%=query.getPlan().getLogicalPlan().toString()%>