From 922e4b9d820d7f1330097f16392e3c0c65f2e23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Wed, 21 May 2014 02:30:38 +0900 Subject: [PATCH 1/7] TAJO-830 --- .../org/apache/tajo/jdbc/TajoResultSet.java | 3 + .../java/org/apache/tajo/conf/TajoConf.java | 5 +- .../planner/BasicLogicalPlanVisitor.java | 5 +- .../tajo/engine/planner/LogicalOptimizer.java | 5 +- .../tajo/engine/planner/LogicalPlan.java | 5 + .../planner/LogicalPlanPreprocessor.java | 6 +- .../tajo/engine/planner/LogicalPlanner.java | 4 +- .../engine/planner/logical/GroupbyNode.java | 10 + .../planner/rewrite/FilterPushDownRule.java | 600 ++++++++++++++++-- .../rewrite/ProjectionPushDownRule.java | 5 + .../org/apache/tajo/TajoTestingCluster.java | 2 + .../tajo/engine/query/TestUnionQuery.java | 53 ++ .../testJoinCoReferredEvalsFilterPushdown.sql | 13 + .../TestSelectQuery/testWhereCond2.sql | 5 +- .../queries/TestUnionQuery/testUnion11.sql | 8 + .../queries/TestUnionQuery/testUnion12.sql | 13 + .../queries/TestUnionQuery/testUnion13.sql | 14 + .../queries/TestUnionQuery/testUnion14.sql | 8 + .../queries/TestUnionQuery/testUnion15.sql | 15 + .../queries/TestUnionQuery/testUnion16.sql | 15 + .../TestSelectQuery/testWhereCond2.result | 6 +- .../results/TestUnionQuery/testUnion11.result | 3 + .../results/TestUnionQuery/testUnion12.result | 6 + .../results/TestUnionQuery/testUnion13.result | 6 + .../results/TestUnionQuery/testUnion14.result | 9 + .../results/TestUnionQuery/testUnion15.result | 5 + .../results/TestUnionQuery/testUnion16.result | 5 + 27 files changed, 764 insertions(+), 70 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestJoinQuery/testJoinCoReferredEvalsFilterPushdown.sql create mode 100644 tajo-core/src/test/resources/queries/TestUnionQuery/testUnion11.sql create mode 100644 tajo-core/src/test/resources/queries/TestUnionQuery/testUnion12.sql create mode 100644 tajo-core/src/test/resources/queries/TestUnionQuery/testUnion13.sql create mode 100644 tajo-core/src/test/resources/queries/TestUnionQuery/testUnion14.sql create mode 100644 tajo-core/src/test/resources/queries/TestUnionQuery/testUnion15.sql create mode 100644 tajo-core/src/test/resources/queries/TestUnionQuery/testUnion16.sql create mode 100644 tajo-core/src/test/resources/results/TestUnionQuery/testUnion11.result create mode 100644 tajo-core/src/test/resources/results/TestUnionQuery/testUnion12.result create mode 100644 tajo-core/src/test/resources/results/TestUnionQuery/testUnion13.result create mode 100644 tajo-core/src/test/resources/results/TestUnionQuery/testUnion14.result create mode 100644 tajo-core/src/test/resources/results/TestUnionQuery/testUnion15.result create mode 100644 tajo-core/src/test/resources/results/TestUnionQuery/testUnion16.result diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java index 336c782619..8595970a7e 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java @@ -87,6 +87,9 @@ private void initScanner() throws IOException { this.totalRow = desc.getStats() != null ? desc.getStats().getNumRows() : INFINITE_ROW_NUM; } + if (totalRow == 0) { + totalRow = INFINITE_ROW_NUM; + } List frags = getFragments(desc.getPath()); scanner = new MergeScanner(conf, desc.getSchema(), desc.getMeta(), frags); diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 65805258d0..cb3ed0d7de 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -249,7 +249,10 @@ public static enum ConfVars { CLI_PRINT_PAUSE_NUM_RECORDS("tajo.cli.print.pause.num.records", 100), CLI_PRINT_PAUSE("tajo.cli.print.pause", true), CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true), - CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"); + CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"), + + //PLANNER + PLANNER_USE_FILTER_PUSHDOWN("tajo.planner.use.filter.pushdown", true) ; public final String varname; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java index 3bffefb752..19b0a9a984 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java @@ -251,8 +251,11 @@ public RESULT visitIntersect(CONTEXT context, LogicalPlan plan, LogicalPlan.Quer public RESULT visitTableSubQuery(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TableSubQueryNode node, Stack stack) throws PlanningException { stack.push(node); + if (node.getPID() == 4) { + int a = 1; + } LogicalPlan.QueryBlock childBlock = plan.getBlock(node.getSubQuery()); - RESULT result = visit(context, plan, childBlock, childBlock.getRoot(), new Stack()); + RESULT result = visit(context, plan, childBlock, childBlock.getRoot(), stack); stack.pop(); return result; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java index 974dc60a5a..3bf70a7b7b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.eval.AlgebraicUtil; import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.planner.graph.DirectedGraphCursor; @@ -54,7 +55,9 @@ public class LogicalOptimizer { public LogicalOptimizer(TajoConf systemConf) { rulesBeforeJoinOpt = new BasicQueryRewriteEngine(); - rulesBeforeJoinOpt.addRewriteRule(new FilterPushDownRule()); + if (systemConf.getBoolVar(ConfVars.PLANNER_USE_FILTER_PUSHDOWN)) { + rulesBeforeJoinOpt.addRewriteRule(new FilterPushDownRule()); + } rulesAfterToJoinOpt = new BasicQueryRewriteEngine(); rulesAfterToJoinOpt.addRewriteRule(new ProjectionPushDownRule()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java index 6be0c6a105..83d482c777 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java @@ -420,6 +420,11 @@ private Column resolveColumnWithoutQualifier(QueryBlock block, return ensureUniqueColumn(candidates); } + LogicalNode blockRootNode = block.getRoot(); + if (blockRootNode != null && blockRootNode.getOutSchema().getColumn(columnRef.getCanonicalName()) != null) { + throw new VerifyException("ERROR: no such a column name "+ columnRef.getCanonicalName()); + } + // Trying to find columns from other relations in other blocks for (QueryBlock eachBlock : queryBlocks.values()) { for (RelationNode rel : eachBlock.getRelations()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index 56863f7a34..a061b5ad69 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -289,11 +289,13 @@ public LogicalNode visitUnion(PreprocessContext ctx, Stack stack, SetOpera LogicalPlan.QueryBlock leftBlock = ctx.plan.newQueryBlock(); PreprocessContext leftContext = new PreprocessContext(ctx, leftBlock); LogicalNode leftChild = visit(leftContext, new Stack(), expr.getLeft()); + leftBlock.setRoot(leftChild); ctx.currentBlock.registerExprWithNode(expr.getLeft(), leftChild); LogicalPlan.QueryBlock rightBlock = ctx.plan.newQueryBlock(); PreprocessContext rightContext = new PreprocessContext(ctx, rightBlock); LogicalNode rightChild = visit(rightContext, new Stack(), expr.getRight()); + rightBlock.setRoot(rightChild); ctx.currentBlock.registerExprWithNode(expr.getRight(), rightChild); UnionNode unionNode = new UnionNode(ctx.plan.newPID()); @@ -363,8 +365,10 @@ public LogicalNode visitTableSubQuery(PreprocessContext ctx, Stack stack, PreprocessContext newContext; // Note: TableSubQuery always has a table name. // SELECT .... FROM (SELECT ...) TB_NAME <- - newContext = new PreprocessContext(ctx, ctx.plan.newQueryBlock()); + QueryBlock queryBlock = ctx.plan.newQueryBlock(); + newContext = new PreprocessContext(ctx, queryBlock); LogicalNode child = super.visitTableSubQuery(newContext, stack, expr); + queryBlock.setRoot(child); // a table subquery should be dealt as a relation. TableSubQueryNode node = ctx.plan.createNode(TableSubQueryNode.class); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index d5d2d479f9..0780d4f68e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -584,7 +584,7 @@ public LogicalNode visitHaving(PlanContext context, Stack stack, Having ex stack.pop(); //////////////////////////////////////////////////////// - HavingNode having = new HavingNode(context.plan.newPID()); + HavingNode having = context.queryBlock.getNodeFromExpr(expr); having.setChild(child); having.setInSchema(child.getOutSchema()); having.setOutSchema(child.getOutSchema()); @@ -1310,7 +1310,7 @@ public LogicalNode visitCreateDatabase(PlanContext context, Stack stack, C @Override public LogicalNode visitDropDatabase(PlanContext context, Stack stack, DropDatabase expr) throws PlanningException { - DropDatabaseNode dropDatabaseNode = context.plan.createNode(DropDatabaseNode.class); + DropDatabaseNode dropDatabaseNode = context.queryBlock.getNodeFromExpr(expr); dropDatabaseNode.init(expr.getDatabaseName(), expr.isIfExists()); return dropDatabaseNode; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java index 828b06dea0..d8d9061030 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java @@ -237,4 +237,14 @@ public PlanString getPlanString() { return planStr; } + + public boolean isAggregationColumn(String simpleName) { + for (int i = groupingColumns.length; i < targets.length; i++) { + if (simpleName.equals(targets[i].getNamedColumn().getSimpleName()) || + simpleName.equals(targets[i].getAlias())) { + return true; + } + } + return false; + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java index 32d4f34240..eb293b9f6c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java @@ -20,19 +20,53 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.exception.InvalidQueryException; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.planner.rewrite.FilterPushDownRule.FilterPushDownContext; import org.apache.tajo.util.TUtil; import java.util.*; -public class FilterPushDownRule extends BasicLogicalPlanVisitor, LogicalNode> implements RewriteRule { +public class FilterPushDownRule extends BasicLogicalPlanVisitor + implements RewriteRule { + private final static Log LOG = LogFactory.getLog(FilterPushDownRule.class); private static final String NAME = "FilterPushDown"; + static class FilterPushDownContext { + Set workingEvals = new HashSet(); + + public void clear() { + workingEvals.clear(); + } + public void setWorkingEvals(Collection workingEvals) { + this.workingEvals.clear(); + this.workingEvals.addAll(workingEvals); + } + public void addWorkingEvals(Collection workingEvals) { + this.workingEvals.addAll(workingEvals); + } + + public void setToOrigin(Map evalMap) { + //evalMap: copy -> origin + List origins = new ArrayList(); + for (EvalNode eval : workingEvals) { + EvalNode origin = evalMap.get(eval); + if (origin != null) { + origins.add(origin); + } + } + setWorkingEvals(origins); + } + } + @Override public String getName() { return NAME; @@ -50,23 +84,40 @@ public boolean isEligible(LogicalPlan plan) { @Override public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { + /* + FilterPushDown rule: processing when visits each node + - If a target which is corresponding on a filter EvalNode's column is not FieldEval, do not PushDown. + - Replace filter EvalNode's column with child node's output column. + If there is no child node's output column, do not PushDown. + - When visit ScanNode, add filter eval to ScanNode's qual + - When visit GroupByNode, Find aggregation column in a filter EvalNode and + . If a parent is HavingNode, add filter eval to parent HavingNode. + . It not, create new HavingNode and set parent's child. + */ + FilterPushDownContext context = new FilterPushDownContext(); for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) { - this.visit(new HashSet(), plan, block, block.getRoot(), new Stack()); + context.clear(); + this.visit(context, plan, block, block.getRoot(), new Stack()); } + if (LOG.isDebugEnabled()) { + LOG.debug("============================================="); + LOG.debug("FilterPushDown Optimized Query: \n" + plan.toString()); + LOG.debug("============================================="); + } return plan; } @Override - public LogicalNode visitFilter(Set cnf, LogicalPlan plan, LogicalPlan.QueryBlock block, + public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, SelectionNode selNode, Stack stack) throws PlanningException { - cnf.addAll(Sets.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(selNode.getQual()))); + context.workingEvals.addAll(Sets.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(selNode.getQual()))); stack.push(selNode); - visit(cnf, plan, block, selNode.getChild(), stack); + visit(context, plan, block, selNode.getChild(), stack); stack.pop(); - if(cnf.size() == 0) { // remove the selection operator if there is no search condition after selection push. + if(context.workingEvals.size() == 0) { // remove the selection operator if there is no search condition after selection push. LogicalNode node = stack.peek(); if (node instanceof UnaryNode) { UnaryNode unary = (UnaryNode) node; @@ -78,16 +129,16 @@ public LogicalNode visitFilter(Set cnf, LogicalPlan plan, LogicalPlan. // check if it can be evaluated here Set matched = TUtil.newHashSet(); - for (EvalNode eachEval : cnf) { + for (EvalNode eachEval : context.workingEvals) { if (LogicalPlanner.checkIfBeEvaluatedAtThis(eachEval, selNode)) { matched.add(eachEval); } } - // if there are search conditions which can be evaluated here, push down them and remove them from cnf. + // if there are search conditions which can be evaluated here, push down them and remove them from context.workingEvals. if (matched.size() > 0) { selNode.setQual(AlgebraicUtil.createSingletonExprFromCNF(matched.toArray(new EvalNode[matched.size()]))); - cnf.removeAll(matched); + context.workingEvals.removeAll(matched); } } @@ -99,7 +150,7 @@ private boolean isOuterJoin(JoinType joinType) { } @Override - public LogicalNode visitJoin(Set cnf, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode joinNode, + public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode joinNode, Stack stack) throws PlanningException { LogicalNode left = joinNode.getRightChild(); LogicalNode right = joinNode.getLeftChild(); @@ -157,7 +208,7 @@ public LogicalNode visitJoin(Set cnf, LogicalPlan plan, LogicalPlan.Qu // retain in this outer join node's JoinQual those selection predicates // related to the outer join's null supplier(s) List matched2 = Lists.newArrayList(); - for (EvalNode eval : cnf) { + for (EvalNode eval : context.workingEvals) { Set columnRefs = EvalTreeUtil.findUniqueColumns(eval); Set tableNames = Sets.newHashSet(); @@ -183,7 +234,7 @@ public LogicalNode visitJoin(Set cnf, LogicalPlan plan, LogicalPlan.Qu } - //merge the retained predicates and establish them in the current outer join node. Then remove them from the cnf + //merge the retained predicates and establish them in the current outer join node. Then remove them from the workingEvals EvalNode qual2 = null; if (matched2.size() > 1) { // merged into one eval tree @@ -197,20 +248,35 @@ public LogicalNode visitJoin(Set cnf, LogicalPlan plan, LogicalPlan.Qu if (qual2 != null) { EvalNode conjQual2 = AlgebraicUtil.createSingletonExprFromCNF(joinNode.getJoinQual(), qual2); joinNode.setJoinQual(conjQual2); - cnf.removeAll(matched2); - } // for the remaining cnf, push it as usual + context.workingEvals.removeAll(matched2); + } // for the remaining context.workingEvals, push it as usual } } if (joinNode.hasJoinQual()) { - cnf.addAll(Sets.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))); + context.addWorkingEvals(Sets.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))); } - visit(cnf, plan, block, left, stack); - visit(cnf, plan, block, right, stack); + List notMatched = new ArrayList(); + // Join's input schema = right child output columns + left child output columns + Map transformedMap = findCanPushdownAndTransform(context, joinNode, left, notMatched, true, + right.getOutSchema().size()); + context.setWorkingEvals(transformedMap.keySet()); + visit(context, plan, block, left, stack); + + context.setToOrigin(transformedMap); + context.addWorkingEvals(notMatched); + + transformedMap = findCanPushdownAndTransform(context, joinNode, right, notMatched, true, 0); + context.setWorkingEvals(new HashSet(transformedMap.keySet())); + + visit(context, plan, block, right, stack); + + context.setToOrigin(transformedMap); + context.addWorkingEvals(notMatched); List matched = Lists.newArrayList(); - for (EvalNode eval : cnf) { + for (EvalNode eval : context.workingEvals) { if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, eval, joinNode, stack.peek().getType() != NodeType.JOIN)) { matched.add(eval); } @@ -232,50 +298,84 @@ public LogicalNode visitJoin(Set cnf, LogicalPlan plan, LogicalPlan.Qu if (joinNode.getJoinType() == JoinType.CROSS) { joinNode.setJoinType(JoinType.INNER); } - cnf.removeAll(matched); + context.workingEvals.removeAll(matched); } return joinNode; } - @Override - public LogicalNode visitTableSubQuery(Set cnf, LogicalPlan plan, LogicalPlan.QueryBlock block, - TableSubQueryNode node, Stack stack) throws PlanningException { - List matched = Lists.newArrayList(); - for (EvalNode eval : cnf) { - if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, node)) { - matched.add(eval); + private Map transformEvalsWidthByPassNode(Collection originEvals, LogicalPlan plan, + LogicalPlan.QueryBlock block, + LogicalNode node, LogicalNode childNode) throws PlanningException { + // transformed -> workingEvals + Map transformedMap = new HashMap(); + + if (originEvals.isEmpty()) { + return transformedMap; + } + + if (node.getType() == NodeType.UNION) { + // Node가 Union이면 Eval 컬럼은 모두 Simple Name이고 Child의 OutSchema의 Simple Name과 모두 매칭 + Schema childOutSchema = childNode.getOutSchema(); + for (EvalNode eval : originEvals) { + EvalNode copy; + try { + copy = (EvalNode) eval.clone(); + } catch (CloneNotSupportedException e) { + throw new PlanningException(e); + } + + Set columns = EvalTreeUtil.findUniqueColumns(copy); + for (Column c : columns) { + Column column = childOutSchema.getColumn(c.getSimpleName()); + if (column == null) { + throw new PlanningException( + "Invalid Filter PushDown on SubQuery: No such a corresponding column '" + + c.getQualifiedName() + " for FilterPushDown(" + eval + "), " + + "(PID=" + node.getPID() + ", Child=" + childNode.getPID() + ")"); + } + EvalTreeUtil.changeColumnRef(copy, c.getSimpleName(), column.getQualifiedName()); + } + + transformedMap.put(copy, eval); } + return transformedMap; } - Map columnMap = new HashMap(); - for (int i = 0; i < node.getInSchema().size(); i++) { - LogicalNode childNode = node.getSubQuery(); - if (childNode.getOutSchema().getColumn(i).hasQualifier()) { - columnMap.put(node.getInSchema().getColumn(i).getQualifiedName(), - childNode.getOutSchema().getColumn(i).getQualifiedName()); - } else { - NamedExprsManager namedExprsMgr = plan.getBlock(node.getSubQuery()).getNamedExprsManager(); - String originalName = namedExprsMgr.getOriginalName(childNode.getOutSchema().getColumn(i) - .getQualifiedName()); - - // We need to consider aliased columns of sub-query. - // Because we can't get original column name for a special occasion. - // For example, if we use an aliased name inside a sub-query and then we use it to where - // condition outside the sub-query, we can't find its original name. - if (originalName != null) { - columnMap.put(node.getInSchema().getColumn(i).getQualifiedName(), originalName); - } else { - columnMap.put(node.getInSchema().getColumn(i).getQualifiedName(), - node.getInSchema().getColumn(i).getQualifiedName()); + if (childNode.getType() == NodeType.UNION) { + // Child가 Union이면 Eval의 컬럼의 Qualifier를 제거하고 반환 + for (EvalNode eval : originEvals) { + EvalNode copy; + try { + copy = (EvalNode) eval.clone(); + } catch (CloneNotSupportedException e) { + throw new PlanningException(e); } + + Set columns = EvalTreeUtil.findUniqueColumns(copy); + for (Column c : columns) { + if (c.hasQualifier()) { + EvalTreeUtil.changeColumnRef(copy, c.getQualifiedName(), c.getSimpleName()); + } + } + + transformedMap.put(copy, eval); } + + return transformedMap; } - Set transformed = new HashSet(); + // node in column -> child out column(가능한 QualifierName) + Map columnMap = new HashMap(); + + for (int i = 0; i < node.getInSchema().size(); i++) { + String inColumnName = node.getInSchema().getColumn(i).getQualifiedName(); + Column childOutColumn = childNode.getOutSchema().getColumn(i); + columnMap.put(inColumnName, childOutColumn.getQualifiedName()); + } // Rename from upper block's one to lower block's one - for (EvalNode matchedEval : matched) { + for (EvalNode matchedEval : originEvals) { EvalNode copy; try { copy = (EvalNode) matchedEval.clone(); @@ -284,31 +384,398 @@ public LogicalNode visitTableSubQuery(Set cnf, LogicalPlan plan, Logic } Set columns = EvalTreeUtil.findUniqueColumns(copy); + boolean allMatched = true; for (Column c : columns) { if (columnMap.containsKey(c.getQualifiedName())) { EvalTreeUtil.changeColumnRef(copy, c.getQualifiedName(), columnMap.get(c.getQualifiedName())); } else { - throw new PlanningException( - "Invalid Filter PushDown on SubQuery: No such a corresponding column '" - + c.getQualifiedName()); + if (childNode.getType() == NodeType.GROUP_BY) { + if (((GroupbyNode) childNode).isAggregationColumn(c.getSimpleName())) { + allMatched = false; + break; + } + } else { + throw new PlanningException( + "Invalid Filter PushDown on SubQuery: No such a corresponding column '" + + c.getQualifiedName() + " for FilterPushDown(" + matchedEval + "), " + + "(PID=" + node.getPID() + ", Child=" + childNode.getPID() + ")" + ); + } } } + if (allMatched) { + transformedMap.put(copy, matchedEval); + } + } - transformed.add(copy); + return transformedMap; + } + + @Override + public LogicalNode visitTableSubQuery(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + TableSubQueryNode node, Stack stack) throws PlanningException { + List matched = Lists.newArrayList(); + for (EvalNode eval : context.workingEvals) { + if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, node)) { + matched.add(eval); + } } - visit(transformed, plan, plan.getBlock(node.getSubQuery())); + // transformed -> workingEvals + Map transformedMap = + transformEvalsWidthByPassNode(matched, plan, block, node, node.getSubQuery()); - cnf.removeAll(matched); + context.setWorkingEvals(new HashSet(transformedMap.keySet())); + visit(context, plan, plan.getBlock(node.getSubQuery())); + + context.setToOrigin(transformedMap); return node; } @Override - public LogicalNode visitScan(Set cnf, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode, + public LogicalNode visitUnion(FilterPushDownContext context, LogicalPlan plan, + LogicalPlan.QueryBlock block, UnionNode unionNode, + Stack stack) throws PlanningException { + LogicalNode leftNode = unionNode.getLeftChild(); + + List origins = new ArrayList(context.workingEvals); + + // transformed -> workingEvals + Map transformedMap = transformEvalsWidthByPassNode(origins, plan, block, unionNode, leftNode); + context.setWorkingEvals(new HashSet(transformedMap.keySet())); + visit(context, plan, plan.getBlock(leftNode)); + + if (!context.workingEvals.isEmpty()) { + errorFilterPushDown(plan, leftNode, context); + } + + LogicalNode rightNode = unionNode.getRightChild(); + transformedMap = transformEvalsWidthByPassNode(origins, plan, block, unionNode, rightNode); + context.setWorkingEvals(new HashSet(transformedMap.keySet())); + visit(context, plan, plan.getBlock(rightNode), rightNode, stack); + + if (!context.workingEvals.isEmpty()) { + errorFilterPushDown(plan, rightNode, context); + } + + // notify all filter matched to upper + context.workingEvals.clear(); + return unionNode; + } + + @Override + public LogicalNode visitProjection(FilterPushDownContext context, + LogicalPlan plan, + LogicalPlan.QueryBlock block, + ProjectionNode projectionNode, + Stack stack) throws PlanningException { + if (projectionNode.getPID() == 7) { + int a = 1; + } + LogicalNode childNode = projectionNode.getChild(); + + List notMatched = new ArrayList(); + + //copy -> origin + Map matched = findCanPushdownAndTransform(context, projectionNode, childNode, notMatched, false, 0); + + context.setWorkingEvals(matched.keySet()); + + stack.push(projectionNode); + LogicalNode current = visit(context, plan, plan.getBlock(childNode), childNode, stack); + stack.pop(); + + // find not matched after visiting child + for (EvalNode eval: context.workingEvals) { + notMatched.add(matched.get(eval)); + } + + EvalNode qual = null; + if (notMatched.size() > 1) { + // merged into one eval tree + qual = AlgebraicUtil.createSingletonExprFromCNF(notMatched.toArray(new EvalNode[notMatched.size()])); + } else if (notMatched.size() == 1) { + // if the number of matched expr is one + qual = notMatched.get(0); + } + + // If there is not matched node add SelectionNode and clear context.workingEvals + if (qual != null) { + SelectionNode selectionNode = plan.createNode(SelectionNode.class); + selectionNode.setInSchema(current.getOutSchema()); + selectionNode.setOutSchema(current.getOutSchema()); + selectionNode.setQual(qual); + block.registerNode(selectionNode); + + projectionNode.setChild(selectionNode); + selectionNode.setChild(current); + } + + //notify all eval matched to upper + context.workingEvals.clear(); + + return current; + } + + private Map findCanPushdownAndTransform(FilterPushDownContext context, Projectable node, + LogicalNode childNode, List notMatched, + boolean ignoreJoin, int columnOffset) throws PlanningException { + // canonical name -> target + Map nodeTargetMap = new HashMap(); + for (Target target : node.getTargets()) { + nodeTargetMap.put(target.getCanonicalName(), target); + } + + // copy -> origin + Map matched = new HashMap(); + + for (EvalNode eval : context.workingEvals) { + if (ignoreJoin && EvalTreeUtil.isJoinQual(eval, true)) { + notMatched.add(eval); + continue; + } + // If all column is field eval, can push down. + Set evalColumns = EvalTreeUtil.findUniqueColumns(eval); + boolean columnMatched = true; + for (Column c : evalColumns) { + Target target = nodeTargetMap.get(c.getQualifiedName()); + if (target == null) { + columnMatched = false; + break; + } + if (target.getEvalTree().getType() != EvalType.FIELD) { + columnMatched = false; + break; + } + } + + if (columnMatched) { + // transform eval column to child's output column + EvalNode copyEvalNode = transformEval(node, childNode, eval, nodeTargetMap, columnOffset); + if (copyEvalNode != null) { + matched.put(copyEvalNode, eval); + } else { + notMatched.add(eval); + } + } else { + notMatched.add(eval); + } + } + + return matched; + } + + private EvalNode transformEval(Projectable node, LogicalNode childNode, EvalNode origin, + Map targetMap, int columnOffset) throws PlanningException { + Schema outputSchema = childNode != null ? childNode.getOutSchema() : node.getInSchema(); + EvalNode copy; + try { + copy = (EvalNode) origin.clone(); + } catch (CloneNotSupportedException e) { + throw new PlanningException(e); + } + Set columns = EvalTreeUtil.findUniqueColumns(copy); + for (Column c: columns) { + Target target = targetMap.get(c.getQualifiedName()); + if (target == null) { + throw new PlanningException( + "Invalid Filter PushDown: No such a corresponding target '" + + c.getQualifiedName() + " for FilterPushDown(" + origin + "), " + + "(PID=" + node.getPID() + ")" + ); + } + EvalNode targetEvalNode = target.getEvalTree(); + if (targetEvalNode.getType() != EvalType.FIELD) { + throw new PlanningException( + "Invalid Filter PushDown: '" + c.getQualifiedName() + "' target is not FieldEval " + + "(PID=" + node.getPID() + ")" + ); + } + + FieldEval fieldEval = (FieldEval)targetEvalNode; + Column targetInputColumn = fieldEval.getColumnRef(); + + int index; + if (targetInputColumn.hasQualifier()) { + index = node.getInSchema().getColumnId(targetInputColumn.getQualifiedName()); + } else { + index = node.getInSchema().getColumnIdByName(targetInputColumn.getQualifiedName()); + } + if (columnOffset > 0) { + index = index - columnOffset; + } + if (index < 0 || index >= outputSchema.size()) { + return null; + } + Column outputColumn = outputSchema.getColumn(index); + + EvalTreeUtil.changeColumnRef(copy, c.getQualifiedName(), outputColumn.getQualifiedName()); + } + + return copy; + } + + /** + * Find aggregation columns in filter eval and add having clause or add HavingNode. + * @param context + * @param plan + * @param block + * @param parentNode If null, having is parent + * @param havingNode If null, projection is parent + * @param groupByNode + * @return matched origin eval + * @throws PlanningException + */ + private List addHavingNode(FilterPushDownContext context, LogicalPlan plan, + LogicalPlan.QueryBlock block, + UnaryNode parentNode, + HavingNode havingNode, + GroupbyNode groupByNode) throws PlanningException { + // find aggregation column + Set groupingColumns = new HashSet(Arrays.asList(groupByNode.getGroupingColumns())); + Set aggrFunctionOutColumns = new HashSet(); + for (Column column : groupByNode.getOutSchema().getColumns()) { + if (!groupingColumns.contains(column)) { + aggrFunctionOutColumns.add(column.getQualifiedName()); + } + } + + List aggrEvalOrigins = new ArrayList(); + List aggrEvals = new ArrayList(); + + for (EvalNode eval : context.workingEvals) { + EvalNode copy = null; + try { + copy = (EvalNode)eval.clone(); + } catch (CloneNotSupportedException e) { + } + boolean isEvalAggrFunction = false; + for (Column evalColumn : EvalTreeUtil.findUniqueColumns(copy)) { + if (aggrFunctionOutColumns.contains(evalColumn.getSimpleName())) { + EvalTreeUtil.changeColumnRef(copy, evalColumn.getQualifiedName(), evalColumn.getSimpleName()); + isEvalAggrFunction = true; + break; + } + } + if (isEvalAggrFunction) { + aggrEvals.add(copy); + aggrEvalOrigins.add(eval); + } + } + + if (aggrEvals.isEmpty()) { + return aggrEvalOrigins; + } + + // transform + + HavingNode workingHavingNode; + if (havingNode != null) { + workingHavingNode = havingNode; + aggrEvals.add(havingNode.getQual()); + } else { + workingHavingNode = plan.createNode(HavingNode.class); + block.registerNode(workingHavingNode); + parentNode.setChild(workingHavingNode); + workingHavingNode.setChild(groupByNode); + } + + EvalNode qual = null; + if (aggrEvals.size() > 1) { + // merged into one eval tree + qual = AlgebraicUtil.createSingletonExprFromCNF(aggrEvals.toArray(new EvalNode[aggrEvals.size()])); + } else if (aggrEvals.size() == 1) { + // if the number of matched expr is one + qual = aggrEvals.get(0); + } + + // If there is not matched node add SelectionNode and clear context.workingEvals + if (qual != null) { + workingHavingNode.setQual(qual); + } + + return aggrEvalOrigins; + } + + @Override + public LogicalNode visitGroupBy(FilterPushDownContext context, LogicalPlan plan, + LogicalPlan.QueryBlock block, GroupbyNode groupbyNode, + Stack stack) throws PlanningException { + LogicalNode parentNode = stack.peek(); + List aggrEvals; + if (parentNode.getType() == NodeType.HAVING) { + aggrEvals = addHavingNode(context, plan, block, null, (HavingNode)parentNode, groupbyNode); + } else { + aggrEvals = addHavingNode(context, plan, block, (UnaryNode)parentNode, null, groupbyNode); + } + + if (aggrEvals != null) { + // remove aggregation eval from conext + context.workingEvals.removeAll(aggrEvals); + } + + List notMatched = new ArrayList(); + // transform + Map tranformed = + findCanPushdownAndTransform(context, groupbyNode,groupbyNode.getChild(), notMatched, false, 0); + + context.setWorkingEvals(tranformed.keySet()); + LogicalNode current = super.visitGroupBy(context, plan, block, groupbyNode, stack); + + context.setToOrigin(tranformed); + context.addWorkingEvals(notMatched); + + return current; + } + + @Override + public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, + LogicalPlan.QueryBlock block, ScanNode scanNode, Stack stack) throws PlanningException { List matched = Lists.newArrayList(); - for (EvalNode eval : cnf) { + + // find partition column and check matching + Set partitionColumns = new HashSet(); + TableDesc table = scanNode.getTableDesc(); + if (table.hasPartition()) { + for (Column c: table.getPartitionMethod().getExpressionSchema().getColumns()) { + partitionColumns.add(c.getQualifiedName()); + } + } + Set partitionEvals = new HashSet(); + for (EvalNode eval : context.workingEvals) { + if (table.hasPartition()) { + Set columns = EvalTreeUtil.findUniqueColumns(eval); + if (columns.size() != 1) { + continue; + } + Column column = columns.iterator().next(); + + if (partitionColumns.contains(column.getSimpleName())) { + EvalNode copy; + try { + copy = (EvalNode) eval.clone(); + } catch (CloneNotSupportedException e) { + throw new PlanningException(e); + } + EvalTreeUtil.changeColumnRef(copy, column.getQualifiedName(), + scanNode.getCanonicalName() + "." + column.getSimpleName()); + matched.add(copy); + partitionEvals.add(eval); + } + } + } + + context.workingEvals.removeAll(partitionEvals); + + List notMatched = new ArrayList(); + + // transform + Map transformed = + findCanPushdownAndTransform(context, scanNode, null, notMatched, true, 0); + + for (EvalNode eval : transformed.keySet()) { if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, scanNode)) { matched.add(eval); } @@ -321,15 +788,32 @@ public LogicalNode visitScan(Set cnf, LogicalPlan plan, LogicalPlan.Qu matched.toArray(new EvalNode[matched.size()])); } else if (matched.size() == 1) { // if the number of matched expr is one - qual = matched.get(0); + qual = matched.iterator().next(); } if (qual != null) { // if a matched qual exists scanNode.setQual(qual); } - cnf.removeAll(matched); + for (EvalNode matchedEval: matched) { + transformed.remove(matchedEval); + } + + context.setToOrigin(transformed); + context.addWorkingEvals(notMatched); return scanNode; } + + private void errorFilterPushDown(LogicalPlan plan, LogicalNode node, FilterPushDownContext context) throws PlanningException { + String notMatchedNodeStr = ""; + String prefix = ""; + for (EvalNode notMatchedNode: context.workingEvals) { + notMatchedNodeStr += prefix + notMatchedNode; + prefix = ", "; + } + throw new PlanningException("FilterPushDown failed cause some filters not matched: " + notMatchedNodeStr + "\n" + + "Error node: " + node.getPlanString() + "\n" + + plan.toString()); + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java index 8e91dca9ed..5df099a48f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java @@ -395,6 +395,9 @@ public LogicalNode visitRoot(Context context, LogicalPlan plan, LogicalPlan.Quer @Override public LogicalNode visitProjection(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, ProjectionNode node, Stack stack) throws PlanningException { + if (node.getPID() == 3) { + int a = 1; + } Context newContext = new Context(context); Target [] targets = node.getTargets(); int targetNum = targets.length; @@ -442,6 +445,8 @@ public LogicalNode visitProjection(Context context, LogicalPlan plan, LogicalPla case TABLE_SUBQUERY: TableSubQueryNode tableSubQueryNode = (TableSubQueryNode) parentNode; tableSubQueryNode.setSubQuery(child); + //KHJ + //block.registerNode(child); break; case STORE: StoreTableNode storeTableNode = (StoreTableNode) parentNode; diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 011ed07ac2..4c878572dc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -101,6 +101,8 @@ void initPropertiesAndConfigs() { this.standbyWorkerMode = conf.getVar(ConfVars.RESOURCE_MANAGER_CLASS) .indexOf(TajoWorkerResourceManager.class.getName()) >= 0; conf.set(CommonTestingUtil.TAJO_TEST, "TRUE"); + +// conf.setBoolean(ConfVars.PLANNER_USE_FILTER_PUSHDOWN.varname, false); } public TajoConf getConfiguration() { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java index a54f6709c2..5845ba845f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java @@ -132,6 +132,59 @@ public final void testUnion10() throws Exception { cleanupQuery(res); } + @Test + public final void testUnion11() throws Exception { + // test filter pushdown + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testUnion12() throws Exception { + // test filter pushdown + // with subquery in union query + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testUnion13() throws Exception { + // test filter pushdown + // with subquery in union query + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testUnion14() throws Exception { + // test filter pushdown + // with group by subquery in union query + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testUnion15() throws Exception { + // test filter pushdown + // with group by out of union query and join in union query + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testUnion16() throws Exception { + // test filter pushdown + // with count distinct out of union query and join in union query + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + @Test public final void testUnionWithSameAliasNames() throws Exception { ResultSet res = executeQuery(); diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testJoinCoReferredEvalsFilterPushdown.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testJoinCoReferredEvalsFilterPushdown.sql new file mode 100644 index 0000000000..680311dab0 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testJoinCoReferredEvalsFilterPushdown.sql @@ -0,0 +1,13 @@ +select * from ( +select + r_regionkey, + n_regionkey, + (r_regionkey + n_regionkey) as plus +from + region, + nation +where + r_regionkey = n_regionkey +order by + r_regionkey, n_regionkey +) where plus > 10 \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testWhereCond2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testWhereCond2.sql index ff5036923a..7fa0d18704 100644 --- a/tajo-core/src/test/resources/queries/TestSelectQuery/testWhereCond2.sql +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testWhereCond2.sql @@ -1,8 +1,9 @@ select * from ( - select a.l_orderkey, count(*) as cnt + select a.l_orderkey, count(*) as cnt, sum(l_extendedprice) as sum1 from lineitem a group by a.l_orderkey + having sum1 > 70000 ) t -where t.cnt > 0 +where t.cnt > 1 order by t.l_orderkey \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion11.sql b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion11.sql new file mode 100644 index 0000000000..ec1a43015f --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion11.sql @@ -0,0 +1,8 @@ +select col1, col2, col3 +from ( + select L_RETURNFLAG as col1, L_EXTENDEDPRICE as col2, concat(L_RECEIPTDATE, L_LINESTATUS) as col3 from lineitem + union all + select P_TYPE as col1, P_RETAILPRICE col2, P_NAME col3 from part +) a +where col3 like '1993%' and col2 > 46796 + diff --git a/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion12.sql b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion12.sql new file mode 100644 index 0000000000..6b6a9ad3c7 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion12.sql @@ -0,0 +1,13 @@ +select col1, col2, col3 +from ( + select + col1, col2, col3 + from + (select + L_RETURNFLAG as col1, L_EXTENDEDPRICE as col2, concat(L_RECEIPTDATE, L_LINESTATUS) as col3 + from + lineitem) b + union all + select P_TYPE as col1, P_RETAILPRICE * 100 col2, concat('1993', P_NAME) col3 from part +) a +where col3 like '1993%' and col2 > 46796 diff --git a/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion13.sql b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion13.sql new file mode 100644 index 0000000000..70b0891a70 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion13.sql @@ -0,0 +1,14 @@ +select col1, col2, col3 +from ( + select + col1, col2, col3 + from + (select + L_RETURNFLAG as col1, L_EXTENDEDPRICE as col2, concat(L_RECEIPTDATE, L_LINESTATUS) as col3 + from + lineitem + where col2 > 46796) b + union all + select P_TYPE as col1, P_RETAILPRICE * 100 col2, concat('1993', P_NAME) col3 from part +) a +where col3 like '1993%' diff --git a/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion14.sql b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion14.sql new file mode 100644 index 0000000000..f47510e360 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion14.sql @@ -0,0 +1,8 @@ +select col1, cnt +from ( + select L_RETURNFLAG as col1, count(*) as cnt from lineitem group by col1 + union all + select cast(n_regionkey as TEXT) as col1, count(*) as cnt from nation group by col1 +) a +where a.cnt > 1 +order by a.col1 \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion15.sql b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion15.sql new file mode 100644 index 0000000000..2e382d0aaa --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion15.sql @@ -0,0 +1,15 @@ +select col1, sum(cnt) +from ( + select col1, cnt + from ( select l_returnflag col1, count(*) cnt from lineitem + join orders on l_orderkey = o_orderkey and o_custkey > 0 + group by l_returnflag) b + where col1 = 'N' + union all + select cast(n_regionkey as TEXT) as col1, count(*) as cnt from nation + where n_regionkey > 2 + group by col1 +) a +where round(cast(a.cnt as FLOAT4)) > 1.0 +group by a.col1 +order by a.col1 diff --git a/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion16.sql b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion16.sql new file mode 100644 index 0000000000..59e9c1f58d --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestUnionQuery/testUnion16.sql @@ -0,0 +1,15 @@ +select col1, sum(cnt) +from ( + select col1, cnt + from ( select l_returnflag col1, count(distinct l_orderkey) cnt from lineitem + join orders on l_orderkey = o_orderkey and o_custkey > 0 + group by l_returnflag) b + where col1 = 'N' + union all + select cast(n_regionkey as TEXT) as col1, count(*) as cnt from nation + where n_regionkey > 2 + group by col1 +) a +where round(cast(a.cnt as FLOAT4)) > 1.0 +group by a.col1 +order by a.col1 diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testWhereCond2.result b/tajo-core/src/test/resources/results/TestSelectQuery/testWhereCond2.result index 32c93ed8b5..c2bee17729 100644 --- a/tajo-core/src/test/resources/results/TestSelectQuery/testWhereCond2.result +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testWhereCond2.result @@ -1,5 +1,3 @@ -l_orderkey,cnt +l_orderkey,cnt,sum1 ------------------------------- -1,2 -2,1 -3,2 \ No newline at end of file +3,2,100854.52 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestUnionQuery/testUnion11.result b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion11.result new file mode 100644 index 0000000000..6e8d2cd657 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion11.result @@ -0,0 +1,3 @@ +col1,col2,col3 +------------------------------- +R,46796.47,1993-11-24F \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestUnionQuery/testUnion12.result b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion12.result new file mode 100644 index 0000000000..c130afa18f --- /dev/null +++ b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion12.result @@ -0,0 +1,6 @@ +col1,col2,col3 +------------------------------- +R,46796.47,1993-11-24F +PROMO BURNISHED COPPER,90100.0,1993goldenrod lavender spring chocolate lace +LARGE BRUSHED BRASS,90200.0,1993blush thistle blue yellow saddle +STANDARD POLISHED BRASS,90300.0,1993spring green yellow purple cornsilk \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestUnionQuery/testUnion13.result b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion13.result new file mode 100644 index 0000000000..c130afa18f --- /dev/null +++ b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion13.result @@ -0,0 +1,6 @@ +col1,col2,col3 +------------------------------- +R,46796.47,1993-11-24F +PROMO BURNISHED COPPER,90100.0,1993goldenrod lavender spring chocolate lace +LARGE BRUSHED BRASS,90200.0,1993blush thistle blue yellow saddle +STANDARD POLISHED BRASS,90300.0,1993spring green yellow purple cornsilk \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestUnionQuery/testUnion14.result b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion14.result new file mode 100644 index 0000000000..3838ab4dab --- /dev/null +++ b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion14.result @@ -0,0 +1,9 @@ +col1,cnt +------------------------------- +0,5 +1,5 +2,5 +3,5 +4,5 +N,3 +R,2 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestUnionQuery/testUnion15.result b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion15.result new file mode 100644 index 0000000000..4e4d9e9b3a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion15.result @@ -0,0 +1,5 @@ +col1,?sum +------------------------------- +3,5 +4,5 +N,3 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestUnionQuery/testUnion16.result b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion16.result new file mode 100644 index 0000000000..38be978381 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestUnionQuery/testUnion16.result @@ -0,0 +1,5 @@ +col1,?sum +------------------------------- +3,5 +4,5 +N,2 \ No newline at end of file From d8b4fa9b046c8a04cbd6fcd7a5b6724068ff8b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Wed, 21 May 2014 02:42:14 +0900 Subject: [PATCH 2/7] Tajo 830: Some filter conditions with a SUBQUERY are removed by optimizer. => Remove unused test code --- .../apache/tajo/engine/planner/BasicLogicalPlanVisitor.java | 3 --- .../apache/tajo/engine/planner/rewrite/FilterPushDownRule.java | 3 --- .../tajo/engine/planner/rewrite/ProjectionPushDownRule.java | 3 --- 3 files changed, 9 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java index 19b0a9a984..331b865745 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java @@ -251,9 +251,6 @@ public RESULT visitIntersect(CONTEXT context, LogicalPlan plan, LogicalPlan.Quer public RESULT visitTableSubQuery(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TableSubQueryNode node, Stack stack) throws PlanningException { stack.push(node); - if (node.getPID() == 4) { - int a = 1; - } LogicalPlan.QueryBlock childBlock = plan.getBlock(node.getSubQuery()); RESULT result = visit(context, plan, childBlock, childBlock.getRoot(), stack); stack.pop(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java index eb293b9f6c..9ba4e27165 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java @@ -470,9 +470,6 @@ public LogicalNode visitProjection(FilterPushDownContext context, LogicalPlan.QueryBlock block, ProjectionNode projectionNode, Stack stack) throws PlanningException { - if (projectionNode.getPID() == 7) { - int a = 1; - } LogicalNode childNode = projectionNode.getChild(); List notMatched = new ArrayList(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java index 5df099a48f..cc7a9fd814 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java @@ -395,9 +395,6 @@ public LogicalNode visitRoot(Context context, LogicalPlan plan, LogicalPlan.Quer @Override public LogicalNode visitProjection(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, ProjectionNode node, Stack stack) throws PlanningException { - if (node.getPID() == 3) { - int a = 1; - } Context newContext = new Context(context); Target [] targets = node.getTargets(); int targetNum = targets.length; From 3f6b6d0918645de5b27ef748f30c3beb733d6894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Sun, 25 May 2014 14:10:57 +0900 Subject: [PATCH 3/7] TAJO-830:Some filter conditions with a SUBQUERY are removed by optimizer(remove the korean comment) --- .../planner/rewrite/FilterPushDownRule.java | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java index 9ba4e27165..af8e44239a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java @@ -117,7 +117,8 @@ public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, visit(context, plan, block, selNode.getChild(), stack); stack.pop(); - if(context.workingEvals.size() == 0) { // remove the selection operator if there is no search condition after selection push. + if(context.workingEvals.size() == 0) { + // remove the selection operator if there is no search condition after selection push. LogicalNode node = stack.peek(); if (node instanceof UnaryNode) { UnaryNode unary = (UnaryNode) node; @@ -135,7 +136,8 @@ public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, } } - // if there are search conditions which can be evaluated here, push down them and remove them from context.workingEvals. + // if there are search conditions which can be evaluated here, + // push down them and remove them from context.workingEvals. if (matched.size() > 0) { selNode.setQual(AlgebraicUtil.createSingletonExprFromCNF(matched.toArray(new EvalNode[matched.size()]))); context.workingEvals.removeAll(matched); @@ -234,7 +236,8 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo } - //merge the retained predicates and establish them in the current outer join node. Then remove them from the workingEvals + // merge the retained predicates and establish them in the current outer join node. + // Then remove them from the workingEvals EvalNode qual2 = null; if (matched2.size() > 1) { // merged into one eval tree @@ -304,9 +307,10 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo return joinNode; } - private Map transformEvalsWidthByPassNode(Collection originEvals, LogicalPlan plan, - LogicalPlan.QueryBlock block, - LogicalNode node, LogicalNode childNode) throws PlanningException { + private Map transformEvalsWidthByPassNode( + Collection originEvals, LogicalPlan plan, + LogicalPlan.QueryBlock block, + LogicalNode node, LogicalNode childNode) throws PlanningException { // transformed -> workingEvals Map transformedMap = new HashMap(); @@ -315,7 +319,7 @@ private Map transformEvalsWidthByPassNode(Collection transformEvalsWidthByPassNode(Collection transformEvalsWidthByPassNode(Collection child out column(가능한 QualifierName) + // node in column -> child out column Map columnMap = new HashMap(); for (int i = 0; i < node.getInSchema().size(); i++) { @@ -475,7 +479,8 @@ public LogicalNode visitProjection(FilterPushDownContext context, List notMatched = new ArrayList(); //copy -> origin - Map matched = findCanPushdownAndTransform(context, projectionNode, childNode, notMatched, false, 0); + Map matched = findCanPushdownAndTransform( + context, projectionNode, childNode, notMatched, false, 0); context.setWorkingEvals(matched.keySet()); @@ -515,9 +520,10 @@ public LogicalNode visitProjection(FilterPushDownContext context, return current; } - private Map findCanPushdownAndTransform(FilterPushDownContext context, Projectable node, - LogicalNode childNode, List notMatched, - boolean ignoreJoin, int columnOffset) throws PlanningException { + private Map findCanPushdownAndTransform( + FilterPushDownContext context, Projectable node, + LogicalNode childNode, List notMatched, + boolean ignoreJoin, int columnOffset) throws PlanningException { // canonical name -> target Map nodeTargetMap = new HashMap(); for (Target target : node.getTargets()) { @@ -802,7 +808,8 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, return scanNode; } - private void errorFilterPushDown(LogicalPlan plan, LogicalNode node, FilterPushDownContext context) throws PlanningException { + private void errorFilterPushDown(LogicalPlan plan, LogicalNode node, + FilterPushDownContext context) throws PlanningException { String notMatchedNodeStr = ""; String prefix = ""; for (EvalNode notMatchedNode: context.workingEvals) { From e87b72725b384496b7e7667e85a5032270ac21af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Mon, 26 May 2014 22:18:40 +0900 Subject: [PATCH 4/7] TAJO-850: OUTER JOIN does not properly handle a NULL. --- .../apache/tajo/engine/eval/EvalTreeUtil.java | 30 +++++++++++++++++++ .../tajo/engine/planner/LogicalPlanner.java | 9 ++++-- .../tajo/engine/query/TestJoinQuery.java | 14 +++++++++ .../testLeftOuterJoinWithNull1.sql | 9 ++++++ .../testLeftOuterJoinWithNull2.sql | 10 +++++++ .../testLeftOuterJoinWithNull1.result | 7 +++++ .../testLeftOuterJoinWithNull2.result | 5 ++++ 7 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull1.sql create mode 100644 tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull2.sql create mode 100644 tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result create mode 100644 tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull2.result diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java index 8982bd54a1..c437d73312 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java @@ -364,6 +364,12 @@ public static Collection findEvalsByType(EvalNode evalNo return (Collection) finder.evalNodes; } + public static Collection findEvalsWithNull(EvalNode evalNode) { + EvalNullFinder finder = new EvalNullFinder(); + finder.visitChild(null, evalNode, new Stack()); + return (Collection) finder.evalNodes; + } + public static class EvalFinder extends BasicEvalNodeVisitor { private EvalType targetType; List evalNodes = TUtil.newList(); @@ -384,6 +390,30 @@ public Object visitChild(Object context, EvalNode evalNode, Stack stac } } + public static class EvalNullFinder extends BasicEvalNodeVisitor { + List evalNodes = TUtil.newList(); + + public EvalNullFinder() { + } + + @Override + public Object visitChild(Object context, EvalNode evalNode, Stack stack) { + super.visitChild(context, evalNode, stack); + + if (evalNode.type == EvalType.FUNCTION) { + FunctionEval functionEval = (FunctionEval)evalNode; + if ("coalesce".equals(functionEval.getName())) { + evalNodes.add(evalNode); + } + } + + if (evalNode.type == EvalType.IS_NULL) { + evalNodes.add(evalNode); + } + return evalNode; + } + } + public static boolean checkIfCanBeConstant(EvalNode evalNode) { return findUniqueColumns(evalNode).size() == 0 && findDistinctAggFunction(evalNode).size() == 0; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index d5d2d479f9..c239c0ea2b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -1552,8 +1552,8 @@ public static boolean checkIfBeEvaluatedAtJoin(QueryBlock block, EvalNode evalNo private static boolean checkIfCaseWhenWithOuterJoinBeEvaluated(QueryBlock block, EvalNode evalNode, boolean isTopMostJoin) { if (block.containsJoinType(JoinType.LEFT_OUTER) || block.containsJoinType(JoinType.RIGHT_OUTER)) { - Collection caseWhenEvals = EvalTreeUtil.findEvalsByType(evalNode, EvalType.CASE); - if (caseWhenEvals.size() > 0 && !isTopMostJoin) { + Collection found = EvalTreeUtil.findEvalsByType(evalNode, EvalType.CASE); + if (found.size() > 0 && !isTopMostJoin) { return false; } } @@ -1581,6 +1581,11 @@ public static boolean checkIfBeEvaluatedAtRelation(QueryBlock block, EvalNode ev if (found.size() > 0) { return false; } + + found = EvalTreeUtil.findEvalsWithNull(evalNode); + if (found.size() > 0) { + return false; + } } return true; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 9bedc10b14..2bf03cd616 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -361,4 +361,18 @@ public final void testJoinOnMultipleDatabasesWithJson() throws Exception { assertResultSet(res); cleanupQuery(res); } + + @Test + public final void testLeftOuterJoinWithNull1() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testLeftOuterJoinWithNull2() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } } diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull1.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull1.sql new file mode 100644 index 0000000000..a54875cff1 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull1.sql @@ -0,0 +1,9 @@ +select + c_custkey, + orders.o_orderkey, + coalesce(orders.o_orderstatus, 'N/A'), + orders.o_orderdate +from + customer left outer join orders on c_custkey = o_orderkey +order by + c_custkey, o_orderkey; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull2.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull2.sql new file mode 100644 index 0000000000..e542c260cb --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull2.sql @@ -0,0 +1,10 @@ +select + c_custkey, + orders.o_orderkey, + coalesce(orders.o_orderstatus, 'N/A'), + orders.o_orderdate +from + customer left outer join orders on c_custkey = o_orderkey +where orders.o_orderdate is not null +order by + c_custkey, o_orderkey; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result new file mode 100644 index 0000000000..4ead3079e6 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result @@ -0,0 +1,7 @@ +c_custkey,o_orderkey,?coalesce,o_orderdate +------------------------------- +1,1,O,1996-01-02 +2,2,O,1996-12-01 +3,3,F,1993-10-14 +4,0,N/A, +5,0,N/A, \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull2.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull2.result new file mode 100644 index 0000000000..72dd50b964 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull2.result @@ -0,0 +1,5 @@ +c_custkey,o_orderkey,?coalesce,o_orderdate +------------------------------- +1,1,O,1996-01-02 +2,2,O,1996-12-01 +3,3,F,1993-10-14 \ No newline at end of file From a88333fa2f05882ab5f850ead56b740512ee48d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Tue, 27 May 2014 23:39:14 +0900 Subject: [PATCH 5/7] TAJO-850: OUTER JOIN does not properly handle a NULL. --- .../org/apache/tajo/client/TajoClient.java | 2 - .../apache/tajo/engine/eval/EvalTreeUtil.java | 9 +- .../tajo/engine/planner/LogicalOptimizer.java | 1 - .../tajo/engine/planner/LogicalPlanner.java | 33 ++++--- .../planner/rewrite/FilterPushDownRule.java | 92 ++++++++----------- .../tajo/engine/query/TestJoinQuery.java | 7 ++ .../queries/TestJoinQuery/oj_table1_ddl.sql | 3 - .../queries/TestJoinQuery/oj_table2_ddl.sql | 3 - .../testLeftOuterJoinWithNull1.sql | 1 + .../testLeftOuterJoinWithNull3.sql | 10 ++ .../testLeftOuterJoinWithNull1.result | 3 - .../testLeftOuterJoinWithNull3.result | 2 + 12 files changed, 83 insertions(+), 83 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull3.sql create mode 100644 tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull3.result diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java index 2f9e1386bd..ea3caa9ae1 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java @@ -345,7 +345,6 @@ public SubmitQueryResponse call(NettyClientBase client) throws ServiceException public ResultSet executeQueryAndGetResult(final String sql) throws ServiceException, IOException { SubmitQueryResponse response = executeQuery(sql); - QueryId queryId = new QueryId(response.getQueryId()); if (response.getIsForwarded()) { if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) { @@ -369,7 +368,6 @@ public ResultSet executeQueryAndGetResult(final String sql) public ResultSet executeJsonQueryAndGetResult(final String json) throws ServiceException, IOException { SubmitQueryResponse response = executeQueryWithJson(json); - QueryId queryId = new QueryId(response.getQueryId()); if (response.getIsForwarded()) { if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java index 09f4cc1645..0c01a5e0d2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java @@ -364,8 +364,8 @@ public static Collection findEvalsByType(EvalNode evalNo return (Collection) finder.evalNodes; } - public static Collection findOuterJoinRelatedEvals(EvalNode evalNode) { - EvalOuterJoinRelatedFinder finder = new EvalOuterJoinRelatedFinder(); + public static Collection findOuterJoinConditionEvals(EvalNode evalNode) { + EvalOuterJoinConditionFinder finder = new EvalOuterJoinConditionFinder(); finder.visitChild(null, evalNode, new Stack()); return (Collection) finder.evalNodes; } @@ -390,10 +390,10 @@ public Object visitChild(Object context, EvalNode evalNode, Stack stac } } - public static class EvalOuterJoinRelatedFinder extends BasicEvalNodeVisitor { + public static class EvalOuterJoinConditionFinder extends BasicEvalNodeVisitor { List evalNodes = TUtil.newList(); - public EvalOuterJoinRelatedFinder() { + public EvalOuterJoinConditionFinder() { } @Override @@ -410,6 +410,7 @@ public Object visitChild(Object context, EvalNode evalNode, Stack stac } else if (evalNode.type == EvalType.IS_NULL) { evalNodes.add(evalNode); } + return evalNode; } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java index 3bf70a7b7b..a436c021fb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java @@ -103,7 +103,6 @@ private void optimizeJoinOrder(LogicalPlan plan, String blockName) throws Planni } else { newJoinNode.setTargets(targets.toArray(new Target[targets.size()])); } - PlannerUtil.replaceNode(plan, block.getRoot(), old, newJoinNode); String optimizedOrder = JoinOrderStringBuilder.buildJoinOrderString(plan, block); block.addPlanHistory("Non-optimized join order: " + originalOrder + " (cost: " + nonOptimizedJoinCost + ")"); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 78e7a97336..420c09966d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -835,8 +835,8 @@ private List getNewlyEvaluatedExprsForJoin(LogicalPlan plan, QueryBlock block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); newlyEvaluatedExprs.add(namedExpr.getAlias()); } - } catch (VerifyException ve) {} catch (PlanningException e) { - e.printStackTrace(); + } catch (VerifyException ve) { + } catch (PlanningException e) { } } return newlyEvaluatedExprs; @@ -1542,22 +1542,25 @@ public static boolean checkIfBeEvaluatedAtJoin(QueryBlock block, EvalNode evalNo // at the topmost join operator. // TODO - It's also valid that case-when is evalauted at the topmost outer operator. // But, how can we know there is no further outer join operator after this node? - if (!checkIfCaseWhenWithOuterJoinBeEvaluated(block, evalNode, isTopMostJoin)) { - return false; + if (containsOuterJoin(block)) { + if (!isTopMostJoin) { + Collection found = EvalTreeUtil.findOuterJoinConditionEvals(evalNode); + if (found.size() > 0) { + return false; + } + } } return true; } - private static boolean checkIfCaseWhenWithOuterJoinBeEvaluated(QueryBlock block, EvalNode evalNode, - boolean isTopMostJoin) { - if (block.containsJoinType(JoinType.LEFT_OUTER) || block.containsJoinType(JoinType.RIGHT_OUTER)) { - Collection found = EvalTreeUtil.findOuterJoinRelatedEvals(evalNode); - if (found.size() > 0) { - return false; - } - } - return true; + public static boolean isOuterJoin(JoinType joinType) { + return joinType == JoinType.LEFT_OUTER || joinType == JoinType.RIGHT_OUTER || joinType==JoinType.FULL_OUTER; + } + + public static boolean containsOuterJoin(QueryBlock block) { + return block.containsJoinType(JoinType.LEFT_OUTER) || block.containsJoinType(JoinType.RIGHT_OUTER) || + block.containsJoinType(JoinType.FULL_OUTER); } /** @@ -1576,8 +1579,8 @@ public static boolean checkIfBeEvaluatedAtRelation(QueryBlock block, EvalNode ev } // Why? - When a {case when} is used with outer join, case when must be evaluated at topmost outer join. - if (block.containsJoinType(JoinType.LEFT_OUTER) || block.containsJoinType(JoinType.RIGHT_OUTER)) { - Collection found = EvalTreeUtil.findOuterJoinRelatedEvals(evalNode); + if (containsOuterJoin(block)) { + Collection found = EvalTreeUtil.findOuterJoinConditionEvals(evalNode); if (found.size() > 0) { return false; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java index b848c862fa..59f529df0d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java @@ -151,10 +151,6 @@ public LogicalNode visitFilter(FilterPushDownContext context, LogicalPlan plan, return selNode; } - private boolean isOuterJoin(JoinType joinType) { - return joinType == JoinType.LEFT_OUTER || joinType == JoinType.RIGHT_OUTER || joinType==JoinType.FULL_OUTER; - } - @Override public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode joinNode, Stack stack) throws PlanningException { @@ -165,7 +161,7 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo // get the two operands of the join operation as well as the join type JoinType joinType = joinNode.getJoinType(); EvalNode joinQual = joinNode.getJoinQual(); - if (joinQual != null && isOuterJoin(joinType)) { + if (joinQual != null && LogicalPlanner.isOuterJoin(joinType)) { BinaryEval binaryEval = (BinaryEval) joinQual; // if both are fields if (binaryEval.getLeftExpr().getType() == EvalType.FIELD && @@ -210,58 +206,45 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo throw new InvalidQueryException("Incorrect Logical Query Plan with regard to outer join"); } } + } + } - // retain in this outer join node's JoinQual those selection predicates - // related to the outer join's null supplier(s) - List matched2 = Lists.newArrayList(); - for (EvalNode eval : context.pushingDownFilters) { - - Set columnRefs = EvalTreeUtil.findUniqueColumns(eval); - Set tableNames = Sets.newHashSet(); - // getting distinct table references - for (Column col : columnRefs) { - if (!tableNames.contains(col.getQualifier())) { - tableNames.add(col.getQualifier()); - } - } + // get evals from ON clause + List onConditions = new ArrayList(); + if (joinNode.hasJoinQual()) { + onConditions.addAll(Sets.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))); + } - //if the predicate involves any of the null suppliers - boolean shouldKeep=false; - Iterator it2 = nullSuppliers.iterator(); - while(it2.hasNext()){ - if(tableNames.contains(it2.next()) == true) { - shouldKeep = true; - } - } + boolean isTopMostJoin = stack.peek().getType() != NodeType.JOIN; - if(shouldKeep == true) { - matched2.add(eval); - } - } - // merge the retained predicates and establish them in the current outer join node. - // Then remove them from the pushingDownFilters - EvalNode qual2 = null; - if (matched2.size() > 1) { - // merged into one eval tree - qual2 = AlgebraicUtil.createSingletonExprFromCNF( - matched2.toArray(new EvalNode[matched2.size()])); - } else if (matched2.size() == 1) { - // if the number of matched expr is one - qual2 = matched2.get(0); - } + List outerJoinConditionEvals = new ArrayList(); + List outerJoinFilterEvalsExcludeJoinCondition = new ArrayList(); + if (LogicalPlanner.isOuterJoin(joinNode.getJoinType())) { + // In the case of top most JOIN, all filters except JOIN condition aren't pushed down. + // That filters are processed by SELECTION NODE. + for (EvalNode eachEval: context.pushingDownFilters) { + if (isTopMostJoin && !EvalTreeUtil.isJoinQual(eachEval, true)) { + outerJoinFilterEvalsExcludeJoinCondition.add(eachEval); - if (qual2 != null) { - EvalNode conjQual2 = AlgebraicUtil.createSingletonExprFromCNF(joinNode.getJoinQual(), qual2); - joinNode.setJoinQual(conjQual2); - context.pushingDownFilters.removeAll(matched2); - } // for the remaining context.pushingDownFilters, push it as usual + } else { + outerJoinConditionEvals.add(eachEval); + } } - } - if (joinNode.hasJoinQual()) { - context.addFiltersTobePushed(Sets.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))); + for (EvalNode eachOnEval: onConditions) { + if (EvalTreeUtil.isJoinQual(eachOnEval, true)) { + // join condition이면 자기가 처리 + outerJoinConditionEvals.add(eachOnEval); + } else { + // TODO pushdown(Null Supplying table) or add join condition(Preserved Row table) + // https://cwiki.apache.org/confluence/display/Hive/OuterJoinBehavior + throw new PlanningException("Currently not support filter condition ON clause in the case of OUTER JOIN."); + } + } + } else { + context.pushingDownFilters.addAll(onConditions); } List notMatched = new ArrayList(); @@ -283,9 +266,13 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo context.addFiltersTobePushed(notMatched); List matched = Lists.newArrayList(); - for (EvalNode eval : context.pushingDownFilters) { - if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, eval, joinNode, stack.peek().getType() != NodeType.JOIN)) { - matched.add(eval); + if(LogicalPlanner.isOuterJoin(joinNode.getJoinType())) { + matched.addAll(outerJoinConditionEvals); + } else { + for (EvalNode eval : context.pushingDownFilters) { + if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, eval, joinNode, isTopMostJoin)) { + matched.add(eval); + } } } @@ -308,6 +295,7 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo context.pushingDownFilters.removeAll(matched); } + context.pushingDownFilters.addAll(outerJoinFilterEvalsExcludeJoinCondition); return joinNode; } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 2bf03cd616..157569cefb 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -375,4 +375,11 @@ public final void testLeftOuterJoinWithNull2() throws Exception { assertResultSet(res); cleanupQuery(res); } + + @Test + public final void testLeftOuterJoinWithNull3() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } } diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table1_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table1_ddl.sql index ea2c954714..a37403cc59 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table1_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table1_ddl.sql @@ -1,6 +1,3 @@ --- Outer Join's Left Table --- It is used in TestJoin::testOuterJoinAndCaseWhen - create external table table1 (id int, name text, score float, type text) using csv with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table2_ddl.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table2_ddl.sql index ac6459ada5..d60b14515d 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table2_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/oj_table2_ddl.sql @@ -1,6 +1,3 @@ --- Outer Join's Left Table --- It is used in TestJoin::testOuterJoinAndCaseWhen - create external table table2 (id int, name text, score float, type text) using csv with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull1.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull1.sql index a54875cff1..5698a74ed5 100644 --- a/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull1.sql +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull1.sql @@ -5,5 +5,6 @@ select orders.o_orderdate from customer left outer join orders on c_custkey = o_orderkey +where o_orderkey is null order by c_custkey, o_orderkey; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull3.sql b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull3.sql new file mode 100644 index 0000000000..32a77503fc --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestJoinQuery/testLeftOuterJoinWithNull3.sql @@ -0,0 +1,10 @@ +select + c_custkey, + orders.o_orderkey, + coalesce(orders.o_orderstatus, 'N/A'), + orders.o_orderdate +from + customer left outer join orders on c_custkey = o_orderkey +where orders.o_orderkey = 100 +order by + c_custkey, o_orderkey; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result index 4ead3079e6..684ed1a3b6 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result @@ -1,7 +1,4 @@ c_custkey,o_orderkey,?coalesce,o_orderdate ------------------------------- -1,1,O,1996-01-02 -2,2,O,1996-12-01 -3,3,F,1993-10-14 4,0,N/A, 5,0,N/A, \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull3.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull3.result new file mode 100644 index 0000000000..efd2e74cf9 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull3.result @@ -0,0 +1,2 @@ +c_custkey,o_orderkey,?coalesce,o_orderdate +------------------------------- From 83a879e9e5f2043b0724f1170b92d3aa2f83ccb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Thu, 29 May 2014 11:15:07 +0900 Subject: [PATCH 6/7] TAJO-850: OUTER JOIN does not properly handle a NULL. --- .../apache/tajo/engine/planner/rewrite/FilterPushDownRule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java index 59f529df0d..ed46e254af 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java @@ -235,7 +235,7 @@ public LogicalNode visitJoin(FilterPushDownContext context, LogicalPlan plan, Lo for (EvalNode eachOnEval: onConditions) { if (EvalTreeUtil.isJoinQual(eachOnEval, true)) { - // join condition이면 자기가 처리 + // If join condition, processing in the JoinNode. outerJoinConditionEvals.add(eachOnEval); } else { // TODO pushdown(Null Supplying table) or add join condition(Preserved Row table) From 0cfb1d4a83bffa0f4c7334d2a0948d140649a20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=ED=98=95=EC=A4=80?= Date: Thu, 29 May 2014 12:13:43 +0900 Subject: [PATCH 7/7] TAJO-850: OUTER JOIN does not properly handle a NULL. --- .../results/TestJoinQuery/testLeftOuterJoinWithNull1.result | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result index 684ed1a3b6..81b907dbd4 100644 --- a/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result +++ b/tajo-core/src/test/resources/results/TestJoinQuery/testLeftOuterJoinWithNull1.result @@ -1,4 +1,4 @@ c_custkey,o_orderkey,?coalesce,o_orderdate ------------------------------- -4,0,N/A, -5,0,N/A, \ No newline at end of file +4,null,N/A,null +5,null,N/A,null \ No newline at end of file