From 8ff4d31297267547cf9111d6ac9fb9ed29e6e859 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sun, 13 Jul 2014 22:25:03 +0900 Subject: [PATCH 01/10] Column Resolver should play different roles at each operation: First refactoring of resolver. --- .../java/org/apache/tajo/catalog/Schema.java | 19 ++- .../tajo/engine/planner/ExprAnnotator.java | 35 +++- .../tajo/engine/planner/LogicalPlan.java | 158 +++++++++++++----- .../planner/LogicalPlanPreprocessor.java | 20 +++ .../tajo/engine/planner/LogicalPlanner.java | 10 +- .../planner/logical/TableSubQueryNode.java | 1 - .../planner/nameresolver/ColumnResolver.java | 27 +++ .../planner/TestLogicalPlanPreprocessor.java | 23 +++ .../tajo/engine/query/TestCaseByCases.java | 8 + .../tajo/engine/query/TestSelectQuery.java | 8 + .../TestCaseByCases/testTAJO917Case1.sql | 13 ++ .../TestSelectQuery/testNonQualifiedNames.sql | 1 + .../TestCaseByCases/testTAJO917Case1.result | 7 + .../testNonQualifiedNames.result | 7 + 14 files changed, 284 insertions(+), 53 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ColumnResolver.java create mode 100644 tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanPreprocessor.java create mode 100644 tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO917Case1.sql create mode 100644 tajo-core/src/test/resources/queries/TestSelectQuery/testNonQualifiedNames.sql create mode 100644 tajo-core/src/test/resources/results/TestCaseByCases/testTAJO917Case1.result create mode 100644 tajo-core/src/test/resources/results/TestSelectQuery/testNonQualifiedNames.result diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java index 35d2fe9225..3aee5d2e3d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java @@ -95,11 +95,22 @@ private void init() { * @param qualifier The qualifier */ public void setQualifier(String qualifier) { + Schema copy = null; + try { + copy = (Schema) clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + + fields.clear(); fieldsByQualifiedName.clear(); - for (int i = 0; i < size(); i++) { - Column column = fields.get(i); - fields.set(i, new Column(qualifier + "." + column.getSimpleName(), column.getDataType())); - fieldsByQualifiedName.put(fields.get(i).getQualifiedName(), i); + fieldsByName.clear(); + + Column newColumn; + for (int i = 0; i < copy.size(); i++) { + Column column = copy.getColumn(i); + newColumn = new Column(qualifier + "." + column.getSimpleName(), column.getDataType()); + addColumn(newColumn); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java index 2c386b2f03..254248768b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java @@ -59,6 +59,13 @@ public class ExprAnnotator extends BaseAlgebraVisitor { private CatalogService catalog; + public static enum ColumnResolvingLevel { + RELS_WITHIN_CURRENT_BLOCK, // finds from only relations + RELS_WITHIN_CURRENT_BLOCK_INCLUDING_SUBEXPRS, // finds from only relations and subexprs + ALL_NAMES_AND_RELS_FIRST, + GLOBAL + } + public ExprAnnotator(CatalogService catalog) { this.catalog = catalog; } @@ -66,16 +73,24 @@ public ExprAnnotator(CatalogService catalog) { static class Context { LogicalPlan plan; LogicalPlan.QueryBlock currentBlock; + ColumnResolvingLevel columnRsvLevel; - public Context(LogicalPlan plan, LogicalPlan.QueryBlock block) { + public Context(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnResolvingLevel colRsvLevel) { this.plan = plan; this.currentBlock = block; + this.columnRsvLevel = colRsvLevel; } } + public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, Expr expr, ColumnResolvingLevel colRsvLevel) + throws PlanningException { + Context context = new Context(plan, block, colRsvLevel); + return AlgebraicUtil.eliminateConstantExprs(visit(context, new Stack(), expr)); + } + public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, Expr expr) throws PlanningException { - Context context = new Context(plan, block); + Context context = new Context(plan, block, ColumnResolvingLevel.GLOBAL); return AlgebraicUtil.eliminateConstantExprs(visit(context, new Stack(), expr)); } @@ -540,7 +555,21 @@ public EvalNode visitSign(Context ctx, Stack stack, SignedExpr expr) throw @Override public EvalNode visitColumnReference(Context ctx, Stack stack, ColumnReferenceExpr expr) throws PlanningException { - Column column = ctx.plan.resolveColumn(ctx.currentBlock, expr); + Column column; + + switch (ctx.columnRsvLevel) { + case GLOBAL: + column = ctx.plan.resolveColumn(ctx.currentBlock, expr); + break; + case RELS_WITHIN_CURRENT_BLOCK: + column = ctx.plan.resolveColumnForRelsWithinCurBlock(ctx.currentBlock, expr, false); + break; + case RELS_WITHIN_CURRENT_BLOCK_INCLUDING_SUBEXPRS: + column = ctx.plan.resolveColumnForRelsWithinCurBlock(ctx.currentBlock, expr, true); + break; + default: + throw new PlanningException("Unsupported column resolving level: " + ctx.columnRsvLevel.name()); + } return new FieldEval(column); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java index 4e1d3137d6..dae278cfdf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.lang.ObjectUtils; +import org.apache.derby.impl.sql.compile.ColumnReference; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.NotThreadSafe; import org.apache.tajo.catalog.CatalogUtil; @@ -32,10 +33,8 @@ import org.apache.tajo.engine.exception.VerifyException; import org.apache.tajo.engine.planner.graph.DirectedGraphCursor; import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.LogicalRootNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.RelationNode; +import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import java.lang.reflect.Constructor; @@ -266,6 +265,111 @@ public SimpleDirectedGraph getQueryBlockGraph() { return queryBlockGraph; } + public Pair normalizeQualifierAndCanonicalName(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { + String qualifier; + String canonicalName; + + if (CatalogUtil.isFQTableName(columnRef.getQualifier())) { + qualifier = columnRef.getQualifier(); + canonicalName = columnRef.getCanonicalName(); + } else { + String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier()); + if (resolvedDatabaseName == null) { + throw new NoSuchColumnException(columnRef.getQualifier()); + } + qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier()); + canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); + } + + return new Pair(qualifier, canonicalName); + } + + Column resolveColumnFromRelationWithinBlock(QueryBlock block, String qualifier, String canonicalName) + throws NoSuchColumnException { + RelationNode relationOp = block.getRelation(qualifier); + + // if a column name is outside of this query block + if (relationOp == null) { + // TODO - nested query can only refer outer query block? or not? + for (QueryBlock eachBlock : queryBlocks.values()) { + if (eachBlock.existsRelation(qualifier)) { + relationOp = eachBlock.getRelation(qualifier); + } + } + } + + // If we cannot find any relation against a qualified column name + if (relationOp == null) { + throw new NoSuchColumnException(canonicalName); + } + + if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) { + String changedName = CatalogUtil.buildFQName( + relationOp.getCanonicalName(), + CatalogUtil.extractSimpleName(canonicalName)); + canonicalName = changedName; + } + + Schema schema = relationOp.getTableSchema(); + Column column = schema.getColumn(canonicalName); + if (column == null) { + throw new NoSuchColumnException(canonicalName); + } + + return column; + } + + Column resolveSubExprReferences(QueryBlock block, ColumnReferenceExpr columnRef) throws NoSuchColumnException { + // Trying to find the column within the current block + if (block.currentNode != null && block.currentNode.getInSchema() != null) { + Column found = block.currentNode.getInSchema().getColumn(columnRef.getCanonicalName()); + if (found != null) { + return found; + } else if (block.getLatestNode() != null) { + found = block.getLatestNode().getOutSchema().getColumn(columnRef.getName()); + if (found != null) { + return found; + } + } + } + throw new NoSuchColumnException(columnRef.getCanonicalName()); + } + + Column resolveColumnForRelsWithinCurBlock(QueryBlock block, ColumnReferenceExpr columnRef, boolean allowNoNameColumn) + throws PlanningException { + String qualifier; + String canonicalName; + + if (allowNoNameColumn && columnRef.getCanonicalName().charAt(0) == LogicalPlan.NONAMED_COLUMN_PREFIX) { + return resolveSubExprReferences(block, columnRef); + } + + if (columnRef.hasQualifier()) { + + Pair normalized = normalizeQualifierAndCanonicalName(block, columnRef); + qualifier = normalized.getFirst(); + canonicalName = normalized.getSecond(); + + return resolveColumnFromRelationWithinBlock(block, qualifier, canonicalName); + } else { + List candidates = TUtil.newList(); + + // It tries to find a full qualified column name from all relations in the current block. + for (RelationNode rel : block.getRelations()) { + Column found = rel.getTableSchema().getColumn(columnRef.getName()); + if (found != null) { + candidates.add(found); + } + } + + if (!candidates.isEmpty()) { + return ensureUniqueColumn(candidates); + } else { + throw new NoSuchColumnException(columnRef.getCanonicalName()); + } + } + } + public String getNormalizedColumnName(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { Column found = resolveColumn(block, columnRef); @@ -307,49 +411,17 @@ public Column resolveColumn(QueryBlock block, ColumnReferenceExpr columnRef) thr } private Column resolveColumnWithQualifier(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { - String qualifier; + final String qualifier; String canonicalName; - String qualifiedName; - - if (CatalogUtil.isFQTableName(columnRef.getQualifier())) { - qualifier = columnRef.getQualifier(); - canonicalName = columnRef.getCanonicalName(); - } else { - String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier()); - if (resolvedDatabaseName == null) { - throw new NoSuchColumnException(columnRef.getQualifier()); - } - qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier()); - canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); - } - qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); + final String qualifiedName; - RelationNode relationOp = block.getRelation(qualifier); + Pair normalized = normalizeQualifierAndCanonicalName(block, columnRef); + qualifier = normalized.getFirst(); + canonicalName = normalized.getSecond(); - // if a column name is outside of this query block - if (relationOp == null) { - // TODO - nested query can only refer outer query block? or not? - for (QueryBlock eachBlock : queryBlocks.values()) { - if (eachBlock.existsRelation(qualifier)) { - relationOp = eachBlock.getRelation(qualifier); - } - } - } - - // If we cannot find any relation against a qualified column name - if (relationOp == null) { - throw new NoSuchColumnException(canonicalName); - } - - if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) { - String changedName = CatalogUtil.buildFQName( - relationOp.getCanonicalName(), - CatalogUtil.extractSimpleName(canonicalName)); - canonicalName = changedName; - } + qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); - Schema schema = relationOp.getTableSchema(); - Column column = schema.getColumn(canonicalName); + Column column = resolveColumnFromRelationWithinBlock(block, qualifier, canonicalName); if (column == null) { throw new NoSuchColumnException(canonicalName); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index 4f1218f2ba..68215f9d99 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -197,6 +197,12 @@ public LogicalNode visitProjection(PreprocessContext ctx, Stack stack, Pro } NamedExpr[] projectTargetExprs = expr.getNamedExprs(); + NameRefInSelectListNormalizer normalizer = new NameRefInSelectListNormalizer(); + + for (int i = 0; i < expr.getNamedExprs().length; i++) { + NamedExpr namedExpr = expr.getNamedExprs()[i]; + normalizer.visit(ctx, new Stack(), namedExpr.getExpr()); + } Target [] targets; targets = new Target[projectTargetExprs.length]; @@ -217,6 +223,8 @@ public LogicalNode visitProjection(PreprocessContext ctx, Stack stack, Pro ProjectionNode projectionNode = ctx.plan.createNode(ProjectionNode.class); projectionNode.setInSchema(child.getOutSchema()); projectionNode.setOutSchema(PlannerUtil.targetToSchema(targets)); + + ctx.currentBlock.setSchema(projectionNode.getOutSchema()); return projectionNode; } @@ -449,4 +457,16 @@ public LogicalNode visitInsert(PreprocessContext ctx, Stack stack, Insert insertNode.setOutSchema(child.getOutSchema()); return insertNode; } + + class NameRefInSelectListNormalizer extends SimpleAlgebraVisitor { + @Override + public Expr visitColumnReference(PreprocessContext ctx, Stack stack, ColumnReferenceExpr expr) + throws PlanningException { + + String normalized = ctx.plan.resolveColumnForRelsWithinCurBlock(ctx.currentBlock, expr, true).getQualifiedName(); + expr.setName(normalized); + + return expr; + } + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index ea517c012c..71de706603 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -51,6 +51,7 @@ import java.util.*; import static org.apache.tajo.algebra.CreateTable.PartitionType; +import static org.apache.tajo.engine.planner.ExprAnnotator.ColumnResolvingLevel; import static org.apache.tajo.engine.planner.ExprNormalizer.ExprNormalizedResult; import static org.apache.tajo.engine.planner.LogicalPlan.BlockType; import static org.apache.tajo.engine.planner.LogicalPlanPreprocessor.PreprocessContext; @@ -407,7 +408,8 @@ private EvalExprNode buildPlanForNoneFromStatement(PlanContext context, Stack stack, Relation e Set newlyEvaluatedExprsReferences = new LinkedHashSet(); for (Iterator iterator = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); iterator.hasNext();) { NamedExpr rawTarget = iterator.next(); +// if (rawTarget.getExpr().getType() == OpType.Column) { +// continue; +// } try { - EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr()); + EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(), + ColumnResolvingLevel.RELS_WITHIN_CURRENT_BLOCK); if (checkIfBeEvaluatedAtRelation(block, evalNode, scanNode)) { block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode); newlyEvaluatedExprsReferences.add(rawTarget.getAlias()); // newly added exr diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java index 4d0090b31e..3c808fcd78 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java @@ -86,7 +86,6 @@ public void setSubQuery(LogicalNode node) { } else { setOutSchema(SchemaUtil.clone(this.subQuery.getOutSchema())); } - getOutSchema().setQualifier(this.tableName); } public LogicalNode getSubQuery() { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ColumnResolver.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ColumnResolver.java new file mode 100644 index 0000000000..77dece2497 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ColumnResolver.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.nameresolver; + +import org.apache.tajo.algebra.ColumnReferenceExpr; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.engine.planner.LogicalPlan; + +public interface ColumnResolver { + public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr col); +} diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanPreprocessor.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanPreprocessor.java new file mode 100644 index 0000000000..e74fce2a2f --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanPreprocessor.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner; + + +public class TestLogicalPlanPreprocessor { +} diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java index 73df4e18a3..45a484262b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCaseByCases.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.query; +import com.google.gson.annotations.Expose; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.TajoConstants; import org.junit.Test; @@ -133,4 +134,11 @@ public final void testTAJO880_3() throws Exception { assertEquals(expected, resultSetToString(res)); cleanupQuery(res); } + + @Test + public final void testTAJO917Case1() throws Exception { + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 639c3efd99..2b0f4af0bf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -48,6 +48,14 @@ public TestSelectQuery() { super(TajoConstants.DEFAULT_DATABASE_NAME); } + @Test + public final void testNonQualifiedNames() throws Exception { + // select l_orderkey, l_partkey from lineitem; + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + @Test public final void testNonFromSelect1() throws Exception { // select upper('abc'); diff --git a/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO917Case1.sql b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO917Case1.sql new file mode 100644 index 0000000000..5b3039ca07 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCaseByCases/testTAJO917Case1.sql @@ -0,0 +1,13 @@ +select + temp.r_regionkey as r_regionkey +from + ( + select + region.r_regionkey as r_regionkey + from + region + ) temp +join + region b +on + temp.r_regionkey = b.r_regionkey; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testNonQualifiedNames.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testNonQualifiedNames.sql new file mode 100644 index 0000000000..0c176b72ad --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testNonQualifiedNames.sql @@ -0,0 +1 @@ +select l_orderkey, l_partkey from lineitem; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO917Case1.result b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO917Case1.result new file mode 100644 index 0000000000..5dbe646243 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestCaseByCases/testTAJO917Case1.result @@ -0,0 +1,7 @@ +r_regionkey +------------------------------- +0 +1 +2 +3 +4 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testNonQualifiedNames.result b/tajo-core/src/test/resources/results/TestSelectQuery/testNonQualifiedNames.result new file mode 100644 index 0000000000..13785365c7 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testNonQualifiedNames.result @@ -0,0 +1,7 @@ +l_orderkey,l_partkey +------------------------------- +1,1 +1,1 +2,2 +3,2 +3,3 \ No newline at end of file From 048c953768da9943c5d87c3f2e680f57e60aa571 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 14 Jul 2014 02:45:23 +0900 Subject: [PATCH 02/10] Column Resolver should play different roles at each operation: Refactored many of createEval to use NameResolver. --- .../tajo/engine/planner/ExprAnnotator.java | 27 ++- .../tajo/engine/planner/ExprNormalizer.java | 6 + .../tajo/engine/planner/LogicalPlan.java | 152 ++++------------- .../planner/LogicalPlanPreprocessor.java | 13 +- .../tajo/engine/planner/LogicalPlanner.java | 36 ++-- ...umnResolver.java => NameResolveLevel.java} | 11 +- .../planner/nameresolver/NameResolver.java | 156 ++++++++++++++++++ .../planner/nameresolver/ResolverByRels.java | 38 +++++ .../ResolverByRelsAndSubExprs.java | 42 +++++ .../ResolverBySubExprsAndRels.java | 42 +++++ 10 files changed, 363 insertions(+), 160 deletions(-) rename tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/{ColumnResolver.java => NameResolveLevel.java} (75%) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java index 254248768b..77f7288754 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java @@ -32,6 +32,8 @@ import org.apache.tajo.engine.function.AggFunction; import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.planner.nameresolver.NameResolver; +import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; import org.apache.tajo.exception.InternalException; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; @@ -59,13 +61,6 @@ public class ExprAnnotator extends BaseAlgebraVisitor { private CatalogService catalog; - public static enum ColumnResolvingLevel { - RELS_WITHIN_CURRENT_BLOCK, // finds from only relations - RELS_WITHIN_CURRENT_BLOCK_INCLUDING_SUBEXPRS, // finds from only relations and subexprs - ALL_NAMES_AND_RELS_FIRST, - GLOBAL - } - public ExprAnnotator(CatalogService catalog) { this.catalog = catalog; } @@ -73,16 +68,17 @@ public ExprAnnotator(CatalogService catalog) { static class Context { LogicalPlan plan; LogicalPlan.QueryBlock currentBlock; - ColumnResolvingLevel columnRsvLevel; + NameResolveLevel columnRsvLevel; - public Context(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnResolvingLevel colRsvLevel) { + public Context(LogicalPlan plan, LogicalPlan.QueryBlock block, NameResolveLevel colRsvLevel) { this.plan = plan; this.currentBlock = block; this.columnRsvLevel = colRsvLevel; } } - public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, Expr expr, ColumnResolvingLevel colRsvLevel) + public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, Expr expr, + NameResolveLevel colRsvLevel) throws PlanningException { Context context = new Context(plan, block, colRsvLevel); return AlgebraicUtil.eliminateConstantExprs(visit(context, new Stack(), expr)); @@ -90,7 +86,7 @@ public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, E public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, Expr expr) throws PlanningException { - Context context = new Context(plan, block, ColumnResolvingLevel.GLOBAL); + Context context = new Context(plan, block, NameResolveLevel.GLOBAL); return AlgebraicUtil.eliminateConstantExprs(visit(context, new Stack(), expr)); } @@ -561,11 +557,10 @@ public EvalNode visitColumnReference(Context ctx, Stack stack, ColumnRefer case GLOBAL: column = ctx.plan.resolveColumn(ctx.currentBlock, expr); break; - case RELS_WITHIN_CURRENT_BLOCK: - column = ctx.plan.resolveColumnForRelsWithinCurBlock(ctx.currentBlock, expr, false); - break; - case RELS_WITHIN_CURRENT_BLOCK_INCLUDING_SUBEXPRS: - column = ctx.plan.resolveColumnForRelsWithinCurBlock(ctx.currentBlock, expr, true); + case RELS_ONLY: + case RELS_AND_SUBEXPRS: + case SUBEXPRS_AND_RELS: + column = NameResolver.resolve(ctx.plan, ctx.currentBlock, expr, ctx.columnRsvLevel); break; default: throw new PlanningException("Unsupported column resolving level: " + ctx.columnRsvLevel.name()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java index 81bbd41247..a38b29c4b9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java @@ -330,6 +330,12 @@ public Expr visitCastExpr(ExprNormalizedResult ctx, Stack stack, CastExpr @Override public Expr visitColumnReference(ExprNormalizedResult ctx, Stack stack, ColumnReferenceExpr expr) throws PlanningException { + + if (ctx.block.isAliasedName(expr.getCanonicalName())) { + String originalName = ctx.block.getOriginalName(expr.getCanonicalName()); + expr.setName(originalName); + return expr; + } // if a column reference is not qualified, it finds and sets the qualified column name. if (!(expr.hasQualifier() && CatalogUtil.isFQTableName(expr.getQualifier()))) { if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName()) && expr.getType() == OpType.Column) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java index dae278cfdf..6e923dd658 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java @@ -19,21 +19,19 @@ package org.apache.tajo.engine.planner; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import org.apache.commons.lang.ObjectUtils; -import org.apache.derby.impl.sql.compile.ColumnReference; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.NotThreadSafe; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.exception.AmbiguousFieldException; import org.apache.tajo.engine.exception.NoSuchColumnException; -import org.apache.tajo.engine.exception.VerifyException; import org.apache.tajo.engine.planner.graph.DirectedGraphCursor; import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.planner.nameresolver.NameResolver; +import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; @@ -265,7 +263,8 @@ public SimpleDirectedGraph getQueryBlockGraph() { return queryBlockGraph; } - public Pair normalizeQualifierAndCanonicalName(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { + public Pair normalizeQualifierAndCanonicalName(QueryBlock block, ColumnReferenceExpr columnRef) + throws PlanningException { String qualifier; String canonicalName; @@ -284,92 +283,6 @@ public Pair normalizeQualifierAndCanonicalName(QueryBlock block, return new Pair(qualifier, canonicalName); } - Column resolveColumnFromRelationWithinBlock(QueryBlock block, String qualifier, String canonicalName) - throws NoSuchColumnException { - RelationNode relationOp = block.getRelation(qualifier); - - // if a column name is outside of this query block - if (relationOp == null) { - // TODO - nested query can only refer outer query block? or not? - for (QueryBlock eachBlock : queryBlocks.values()) { - if (eachBlock.existsRelation(qualifier)) { - relationOp = eachBlock.getRelation(qualifier); - } - } - } - - // If we cannot find any relation against a qualified column name - if (relationOp == null) { - throw new NoSuchColumnException(canonicalName); - } - - if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) { - String changedName = CatalogUtil.buildFQName( - relationOp.getCanonicalName(), - CatalogUtil.extractSimpleName(canonicalName)); - canonicalName = changedName; - } - - Schema schema = relationOp.getTableSchema(); - Column column = schema.getColumn(canonicalName); - if (column == null) { - throw new NoSuchColumnException(canonicalName); - } - - return column; - } - - Column resolveSubExprReferences(QueryBlock block, ColumnReferenceExpr columnRef) throws NoSuchColumnException { - // Trying to find the column within the current block - if (block.currentNode != null && block.currentNode.getInSchema() != null) { - Column found = block.currentNode.getInSchema().getColumn(columnRef.getCanonicalName()); - if (found != null) { - return found; - } else if (block.getLatestNode() != null) { - found = block.getLatestNode().getOutSchema().getColumn(columnRef.getName()); - if (found != null) { - return found; - } - } - } - throw new NoSuchColumnException(columnRef.getCanonicalName()); - } - - Column resolveColumnForRelsWithinCurBlock(QueryBlock block, ColumnReferenceExpr columnRef, boolean allowNoNameColumn) - throws PlanningException { - String qualifier; - String canonicalName; - - if (allowNoNameColumn && columnRef.getCanonicalName().charAt(0) == LogicalPlan.NONAMED_COLUMN_PREFIX) { - return resolveSubExprReferences(block, columnRef); - } - - if (columnRef.hasQualifier()) { - - Pair normalized = normalizeQualifierAndCanonicalName(block, columnRef); - qualifier = normalized.getFirst(); - canonicalName = normalized.getSecond(); - - return resolveColumnFromRelationWithinBlock(block, qualifier, canonicalName); - } else { - List candidates = TUtil.newList(); - - // It tries to find a full qualified column name from all relations in the current block. - for (RelationNode rel : block.getRelations()) { - Column found = rel.getTableSchema().getColumn(columnRef.getName()); - if (found != null) { - candidates.add(found); - } - } - - if (!candidates.isEmpty()) { - return ensureUniqueColumn(candidates); - } else { - throw new NoSuchColumnException(columnRef.getCanonicalName()); - } - } - } - public String getNormalizedColumnName(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { Column found = resolveColumn(block, columnRef); @@ -421,7 +334,8 @@ private Column resolveColumnWithQualifier(QueryBlock block, ColumnReferenceExpr qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); - Column column = resolveColumnFromRelationWithinBlock(block, qualifier, canonicalName); + // Column column = resolveColumnFromRelationWithinBlock(block, qualifier, canonicalName);; + Column column = NameResolver.resolve(this, block, columnRef, NameResolveLevel.RELS_ONLY); if (column == null) { throw new NoSuchColumnException(canonicalName); } @@ -456,7 +370,7 @@ private Column resolveColumnWithQualifier(QueryBlock block, ColumnReferenceExpr } } if (!candidates.isEmpty()) { - return ensureUniqueColumn(candidates); + return NameResolver.ensureUniqueColumn(candidates); } } @@ -477,7 +391,7 @@ private Column resolveColumnWithoutQualifier(QueryBlock block, } if (!candidates.isEmpty()) { - return ensureUniqueColumn(candidates); + return NameResolver.ensureUniqueColumn(candidates); } // Trying to find the column within the current block @@ -505,7 +419,7 @@ private Column resolveColumnWithoutQualifier(QueryBlock block, } } if (!candidates.isEmpty()) { - return ensureUniqueColumn(candidates); + return NameResolver.ensureUniqueColumn(candidates); } // This is an exception case. It means that there are some bugs in other parts. @@ -525,7 +439,7 @@ private Column resolveColumnWithoutQualifier(QueryBlock block, } if (!candidates.isEmpty()) { - return ensureUniqueColumn(candidates); + return NameResolver.ensureUniqueColumn(candidates); } // Trying to find columns from schema in current block. @@ -537,33 +451,12 @@ private Column resolveColumnWithoutQualifier(QueryBlock block, } if (!candidates.isEmpty()) { - return ensureUniqueColumn(candidates); + return NameResolver.ensureUniqueColumn(candidates); } throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName()); } - private static Column ensureUniqueColumn(List candidates) - throws VerifyException { - if (candidates.size() == 1) { - return candidates.get(0); - } else if (candidates.size() > 2) { - StringBuilder sb = new StringBuilder(); - boolean first = true; - for (Column column : candidates) { - if (first) { - first = false; - } else { - sb.append(", "); - } - sb.append(column); - } - throw new AmbiguousFieldException("Ambiguous Column Name: " + sb.toString()); - } else { - return null; - } - } - public String getQueryGraphAsString() { StringBuilder sb = new StringBuilder(); @@ -673,7 +566,8 @@ public class QueryBlock { // transient states private final Map canonicalNameToRelationMap = TUtil.newHashMap(); - private final Map> aliasMap = TUtil.newHashMap(); + private final Map> relationAliasMap = TUtil.newHashMap(); + private final Map columnAliasMap = TUtil.newHashMap(); private final Map> operatorToExprMap = TUtil.newHashMap(); private final List relationList = TUtil.newList(); private boolean hasWindowFunction = false; @@ -741,7 +635,7 @@ public boolean existsRelation(String name) { } public boolean isAlreadyRenamedTableName(String name) { - return aliasMap.containsKey(name); + return relationAliasMap.containsKey(name); } public RelationNode getRelation(String name) { @@ -749,8 +643,8 @@ public RelationNode getRelation(String name) { return canonicalNameToRelationMap.get(name); } - if (aliasMap.containsKey(name)) { - return canonicalNameToRelationMap.get(aliasMap.get(name).get(0)); + if (relationAliasMap.containsKey(name)) { + return canonicalNameToRelationMap.get(relationAliasMap.get(name).get(0)); } return null; @@ -758,7 +652,7 @@ public RelationNode getRelation(String name) { public void addRelation(RelationNode relation) { if (relation.hasAlias()) { - TUtil.putToNestedList(aliasMap, relation.getTableName(), relation.getCanonicalName()); + TUtil.putToNestedList(relationAliasMap, relation.getTableName(), relation.getCanonicalName()); } canonicalNameToRelationMap.put(relation.getCanonicalName(), relation); relationList.add(relation); @@ -772,6 +666,18 @@ public boolean hasTableExpression() { return this.canonicalNameToRelationMap.size() > 0; } + public void addColumnAlias(String original, String alias) { + columnAliasMap.put(alias, original); + } + + public boolean isAliasedName(String alias) { + return columnAliasMap.containsKey(alias); + } + + public String getOriginalName(String alias) { + return columnAliasMap.get(alias); + } + public void setSchema(Schema schema) { this.schema = schema; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index 68215f9d99..cec1738d6b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -26,6 +26,8 @@ import org.apache.tajo.engine.exception.NoSuchColumnException; import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.planner.nameresolver.NameResolver; +import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.master.session.Session; import org.apache.tajo.util.TUtil; @@ -202,6 +204,11 @@ public LogicalNode visitProjection(PreprocessContext ctx, Stack stack, Pro for (int i = 0; i < expr.getNamedExprs().length; i++) { NamedExpr namedExpr = expr.getNamedExprs()[i]; normalizer.visit(ctx, new Stack(), namedExpr.getExpr()); + + if (namedExpr.getExpr().getType() == OpType.Column && namedExpr.hasAlias()) { + ctx.currentBlock.addColumnAlias(((ColumnReferenceExpr)namedExpr.getExpr()).getCanonicalName(), + namedExpr.getAlias()); + } } Target [] targets; @@ -275,7 +282,8 @@ public LogicalNode visitGroupBy(PreprocessContext ctx, Stack stack, Aggreg for (int i = 0; i < finalTargetNum; i++) { NamedExpr namedExpr = projection.getNamedExprs()[i]; - EvalNode evalNode = annotator.createEvalNode(ctx.plan, ctx.currentBlock, namedExpr.getExpr()); + EvalNode evalNode = annotator.createEvalNode(ctx.plan, ctx.currentBlock, namedExpr.getExpr(), + NameResolveLevel.SUBEXPRS_AND_RELS); if (namedExpr.hasAlias()) { targets[i] = new Target(evalNode, namedExpr.getAlias()); @@ -463,7 +471,8 @@ class NameRefInSelectListNormalizer extends SimpleAlgebraVisitor stack, ColumnReferenceExpr expr) throws PlanningException { - String normalized = ctx.plan.resolveColumnForRelsWithinCurBlock(ctx.currentBlock, expr, true).getQualifiedName(); + String normalized = NameResolver.resolve(ctx.plan, ctx.currentBlock, expr, + NameResolveLevel.RELS_AND_SUBEXPRS).getQualifiedName(); expr.setName(normalized); return expr; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 71de706603..a975dfe2f0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -40,6 +40,7 @@ import org.apache.tajo.engine.exception.VerifyException; import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule; import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.master.session.Session; @@ -51,7 +52,7 @@ import java.util.*; import static org.apache.tajo.algebra.CreateTable.PartitionType; -import static org.apache.tajo.engine.planner.ExprAnnotator.ColumnResolvingLevel; + import static org.apache.tajo.engine.planner.ExprNormalizer.ExprNormalizedResult; import static org.apache.tajo.engine.planner.LogicalPlan.BlockType; import static org.apache.tajo.engine.planner.LogicalPlanPreprocessor.PreprocessContext; @@ -261,7 +262,8 @@ private void setRawTargets(PlanContext context, Target[] targets, String[] refer Target [] rawTargets = new Target[projection.getNamedExprs().length]; for (int i = 0; i < projection.getNamedExprs().length; i++) { NamedExpr namedExpr = projection.getNamedExprs()[i]; - EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr()); + EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), + NameResolveLevel.SUBEXPRS_AND_RELS); rawTargets[i] = new Target(evalNode, referenceNames[i]); } // it's for debugging or unit testing @@ -409,7 +411,7 @@ private EvalExprNode buildPlanForNoneFromStatement(PlanContext context, Stack it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr rawTarget = it.next(); try { - EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr()); + EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(), + NameResolveLevel.SUBEXPRS_AND_RELS); if (evalNode.getType() == EvalType.WINDOW_FUNCTION) { winFuncRefs.add(rawTarget.getAlias()); winFuncs.add((WindowFunctionEval) evalNode); @@ -680,7 +683,8 @@ private LogicalNode insertGroupbyNode(PlanContext context, LogicalNode child, St try { // check if at least distinct aggregation function includeDistinctFunction |= PlannerUtil.existsDistinctAggregationFunction(rawTarget.getExpr()); - EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr()); + EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(), + NameResolveLevel.SUBEXPRS_AND_RELS); if (evalNode.getType() == EvalType.AGG_FUNCTION) { aggEvalNames.add(rawTarget.getAlias()); aggEvals.add((AggregationFunctionCallEval) evalNode); @@ -714,7 +718,8 @@ public LimitNode visitLimit(PlanContext context, Stack stack, Limit limit) EvalNode firstFetNum; LogicalNode child; if (limit.getFetchFirstNum().getType() == OpType.Literal) { - firstFetNum = exprAnnotator.createEvalNode(context.plan, block, limit.getFetchFirstNum()); + firstFetNum = exprAnnotator.createEvalNode(context.plan, block, limit.getFetchFirstNum(), + NameResolveLevel.RELS_ONLY); //////////////////////////////////////////////////////// // Visit and Build Child Plan @@ -741,7 +746,8 @@ public LimitNode visitLimit(PlanContext context, Stack stack, Limit limit) firstFetNum = block.namedExprsMgr.getTarget(referName).getEvalTree(); } else { NamedExpr namedExpr = block.namedExprsMgr.getNamedExpr(referName); - firstFetNum = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr()); + firstFetNum = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr(), + NameResolveLevel.SUBEXPRS_AND_RELS); block.namedExprsMgr.markAsEvaluated(referName, firstFetNum); } } @@ -837,7 +843,8 @@ public LogicalNode visitHaving(PlanContext context, Stack stack, Having ex havingCondition = block.namedExprsMgr.getTarget(referName).getEvalTree(); } else { NamedExpr namedExpr = block.namedExprsMgr.getNamedExpr(referName); - havingCondition = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr()); + havingCondition = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr(), + NameResolveLevel.SUBEXPRS_AND_RELS); block.namedExprsMgr.markAsEvaluated(referName, havingCondition); } @@ -904,7 +911,8 @@ public LogicalNode visitGroupBy(PlanContext context, Stack stack, Aggregat NamedExpr namedExpr = iterator.next(); try { includeDistinctFunction |= PlannerUtil.existsDistinctAggregationFunction(namedExpr.getExpr()); - EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, namedExpr.getExpr()); + EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, namedExpr.getExpr(), + NameResolveLevel.SUBEXPRS_AND_RELS); if (evalNode.getType() == EvalType.AGG_FUNCTION) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); aggEvalNames.add(namedExpr.getAlias()); @@ -990,7 +998,8 @@ public SelectionNode visitFilter(PlanContext context, Stack stack, Selecti selectionNode.setOutSchema(child.getOutSchema()); // Create EvalNode for a search condition. - EvalNode searchCondition = exprAnnotator.createEvalNode(context.plan, block, selection.getQual()); + EvalNode searchCondition = exprAnnotator.createEvalNode(context.plan, block, selection.getQual(), + NameResolveLevel.RELS_AND_SUBEXPRS); EvalNode simplified = AlgebraicUtil.eliminateConstantExprs(searchCondition); // set selection condition selectionNode.setQual(simplified); @@ -1199,7 +1208,7 @@ public ScanNode visitRelation(PlanContext context, Stack stack, Relation e // } try { EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(), - ColumnResolvingLevel.RELS_WITHIN_CURRENT_BLOCK); + NameResolveLevel.RELS_ONLY); if (checkIfBeEvaluatedAtRelation(block, evalNode, scanNode)) { block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode); newlyEvaluatedExprsReferences.add(rawTarget.getAlias()); // newly added exr @@ -1214,8 +1223,9 @@ public ScanNode visitRelation(PlanContext context, Stack stack, Relation e // The fact the some expr is included in newlyEvaluatedExprsReferences means that it is already evaluated. // So, we get a raw expression and then creates a target. for (String reference : newlyEvaluatedExprsReferences) { - NamedExpr refrrer = block.namedExprsMgr.getNamedExpr(reference); - EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, refrrer.getExpr()); + NamedExpr refrer = block.namedExprsMgr.getNamedExpr(reference); + EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, refrer.getExpr(), + NameResolveLevel.RELS_ONLY); targets.add(new Target(evalNode, reference)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ColumnResolver.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java similarity index 75% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ColumnResolver.java rename to tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java index 77dece2497..caaedd06d0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ColumnResolver.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java @@ -18,10 +18,9 @@ package org.apache.tajo.engine.planner.nameresolver; -import org.apache.tajo.algebra.ColumnReferenceExpr; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.planner.LogicalPlan; - -public interface ColumnResolver { - public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr col); +public enum NameResolveLevel { + RELS_ONLY, // finds from only relations + RELS_AND_SUBEXPRS, // finds from only relations and subexprs + SUBEXPRS_AND_RELS, + GLOBAL } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java new file mode 100644 index 0000000000..3c0f729df9 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.nameresolver; + +import com.google.common.collect.Maps; +import org.apache.tajo.algebra.ColumnReferenceExpr; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.engine.exception.AmbiguousFieldException; +import org.apache.tajo.engine.exception.NoSuchColumnException; +import org.apache.tajo.engine.exception.VerifyException; +import org.apache.tajo.engine.planner.LogicalPlan; +import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.engine.planner.logical.RelationNode; +import org.apache.tajo.util.Pair; +import org.apache.tajo.util.TUtil; + +import java.util.List; +import java.util.Map; + +public abstract class NameResolver { + + public static Map resolverMap = Maps.newHashMap(); + + static { + resolverMap.put(NameResolveLevel.RELS_ONLY, new ResolverByRels()); + resolverMap.put(NameResolveLevel.RELS_AND_SUBEXPRS, new ResolverByRelsAndSubExprs()); + resolverMap.put(NameResolveLevel.SUBEXPRS_AND_RELS, new ResolverBySubExprsAndRels()); + } + + public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column, + NameResolveLevel level) throws PlanningException { + if (!resolverMap.containsKey(level)) { + throw new PlanningException("Unsupported name resolving level"); + } + return resolverMap.get(level).resolve(plan, block, column); + } + + public static Column ensureUniqueColumn(List candidates) + throws VerifyException { + if (candidates.size() == 1) { + return candidates.get(0); + } else if (candidates.size() > 2) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Column column : candidates) { + if (first) { + first = false; + } else { + sb.append(", "); + } + sb.append(column); + } + throw new AmbiguousFieldException("Ambiguous Column Name: " + sb.toString()); + } else { + return null; + } + } + + /** + * It tries to resolve the variable name from relations within a given query block. + * If a variable name is qualified, it tries to resolve the name from the relation corresponding to the qualifier. + * + * @param plan + * @param block + * @param columnRef + * @return + * @throws PlanningException + */ + public static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef) throws PlanningException { + String qualifier; + String canonicalName; + + if (columnRef.hasQualifier()) { + Pair normalized = plan.normalizeQualifierAndCanonicalName(block, columnRef); + qualifier = normalized.getFirst(); + canonicalName = normalized.getSecond(); + + RelationNode relationOp = block.getRelation(qualifier); + + // If we cannot find any relation against a qualified column name + if (relationOp == null) { + throw null; + } + + // Please consider a query case: + // select lineitem.l_orderkey from lineitem a order by lineitem.l_orderkey; + // + // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still can be used. + // The below code makes it available. Otherwise, it cannot find any match in the relation schema. + if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) { + canonicalName = + CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName)); + } + + Schema schema = relationOp.getTableSchema(); + Column column = schema.getColumn(canonicalName); + + return column; + } else { + List candidates = TUtil.newList(); + + // It tries to find a full qualified column name from all relations in the current block. + for (RelationNode rel : block.getRelations()) { + Column found = rel.getTableSchema().getColumn(columnRef.getName()); + if (found != null) { + candidates.add(found); + } + } + + if (!candidates.isEmpty()) { + return ensureUniqueColumn(candidates); + } else { + return null; + } + } + } + + public static Column resolveSubExprReferences(LogicalPlan plan, LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef) throws NoSuchColumnException { + // Trying to find the column within the current block + if (block.getCurrentNode() != null && block.getCurrentNode().getInSchema() != null) { + Column found = block.getCurrentNode().getInSchema().getColumn(columnRef.getCanonicalName()); + if (found != null) { + return found; + } else if (block.getLatestNode() != null) { + found = block.getLatestNode().getOutSchema().getColumn(columnRef.getName()); + if (found != null) { + return found; + } + } + } + return null; + } + + abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) + throws PlanningException; +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java new file mode 100644 index 0000000000..9713e5251a --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.nameresolver; + +import org.apache.tajo.algebra.ColumnReferenceExpr; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.engine.exception.NoSuchColumnException; +import org.apache.tajo.engine.planner.LogicalPlan; +import org.apache.tajo.engine.planner.PlanningException; + +public class ResolverByRels extends NameResolver { + @Override + public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) + throws PlanningException { + + Column column = resolveFromRelsWithinBlock(plan, block, columnRef); + if (column == null) { + throw new NoSuchColumnException(columnRef.getCanonicalName()); + } + return column; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java new file mode 100644 index 0000000000..4009c2e199 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.nameresolver; + +import org.apache.tajo.algebra.ColumnReferenceExpr; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.engine.exception.NoSuchColumnException; +import org.apache.tajo.engine.planner.LogicalPlan; +import org.apache.tajo.engine.planner.PlanningException; + +public class ResolverByRelsAndSubExprs extends NameResolver { + @Override + public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) + throws PlanningException { + + Column column = resolveFromRelsWithinBlock(plan, block, columnRef); + if (column == null) { + column = resolveSubExprReferences(plan, block, columnRef); + } + + if (column == null) { + throw new NoSuchColumnException(columnRef.getCanonicalName()); + } + return column; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java new file mode 100644 index 0000000000..6908a7810d --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.nameresolver; + +import org.apache.tajo.algebra.ColumnReferenceExpr; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.engine.exception.NoSuchColumnException; +import org.apache.tajo.engine.planner.LogicalPlan; +import org.apache.tajo.engine.planner.PlanningException; + +public class ResolverBySubExprsAndRels extends NameResolver { + @Override + public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) + throws PlanningException { + + Column column = resolveSubExprReferences(plan, block, columnRef); + if (column == null) { + column = resolveFromRelsWithinBlock(plan, block, columnRef); + } + + if (column == null) { + throw new NoSuchColumnException(columnRef.getCanonicalName()); + } + return column; + } +} From cce2a566c1bca84d19e328d3b920c3b96af7bfe9 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 14 Jul 2014 03:28:29 +0900 Subject: [PATCH 03/10] Column Resolver should play different roles at each operation: Refactored CreateEval() except for join. --- .../apache/tajo/engine/planner/ExprAnnotator.java | 6 ------ .../apache/tajo/engine/planner/LogicalPlanner.java | 12 +++++++----- .../apache/tajo/engine/eval/TestEvalTreeUtil.java | 4 +++- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java index 77f7288754..d958e027f3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java @@ -84,12 +84,6 @@ public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, E return AlgebraicUtil.eliminateConstantExprs(visit(context, new Stack(), expr)); } - public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, Expr expr) - throws PlanningException { - Context context = new Context(plan, block, NameResolveLevel.GLOBAL); - return AlgebraicUtil.eliminateConstantExprs(visit(context, new Stack(), expr)); - } - public static void assertEval(boolean condition, String message) throws PlanningException { if (!condition) { throw new PlanningException(message); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index a975dfe2f0..9334a7364c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -386,7 +386,7 @@ private EvalExprNode buildPlanForNoneFromStatement(PlanContext context, Stack stack, Join join) // Create EvalNode for a search condition. EvalNode joinCondition = null; if (join.hasQual()) { - EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, join.getQual()); + EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, join.getQual(), + NameResolveLevel.GLOBAL); joinCondition = AlgebraicUtil.eliminateConstantExprs(evalNode); } @@ -1082,7 +1083,7 @@ private List getNewlyEvaluatedExprsForJoin(LogicalPlan plan, QueryBlock for (Iterator it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr namedExpr = it.next(); try { - evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr()); + evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolveLevel.GLOBAL); if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, evalNode, joinNode, stack.peek().getType() != OpType.Join)) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); newlyEvaluatedExprs.add(namedExpr.getAlias()); @@ -1152,7 +1153,7 @@ private LogicalNode createCartesianProduct(PlanContext context, LogicalNode left for (Iterator it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr namedExpr = it.next(); try { - evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr()); + evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolveLevel.GLOBAL); if (EvalTreeUtil.findDistinctAggFunction(evalNode).size() == 0) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); newlyEvaluatedExprs.add(namedExpr.getAlias()); @@ -1283,7 +1284,8 @@ public TableSubQueryNode visitTableSubQuery(PlanContext context, Stack sta Set newlyEvaluatedExprs = TUtil.newHashSet(); for (NamedExpr rawTarget : block.namedExprsMgr.getAllNamedExprs()) { try { - EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr()); + EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(), + NameResolveLevel.RELS_ONLY); if (checkIfBeEvaluatedAtRelation(block, evalNode, subQueryNode)) { block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode); newlyEvaluatedExprs.add(rawTarget.getAlias()); // newly added exr diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index 92770ec09f..e66c67c0bf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -39,6 +39,7 @@ import org.apache.tajo.engine.planner.Target; import org.apache.tajo.engine.planner.logical.GroupbyNode; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; import org.apache.tajo.exception.InternalException; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; @@ -154,7 +155,8 @@ public static EvalNode getRootSelection(String query) throws PlanningException { } Selection selection = plan.getRootBlock().getSingletonExpr(OpType.Filter); - return planner.getExprAnnotator().createEvalNode(plan, plan.getRootBlock(), selection.getQual()); + return planner.getExprAnnotator().createEvalNode(plan, plan.getRootBlock(), selection.getQual(), + NameResolveLevel.RELS_AND_SUBEXPRS); } @Test From 2d1a1d81631056711d5ff91d92f0f934c9561525 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 14 Jul 2014 11:15:29 +0900 Subject: [PATCH 04/10] Column Resolver should play different roles at each operation: Completely extracted the column resolver from LogicalPlan. --- .../tajo/engine/planner/ExprNormalizer.java | 5 +- .../tajo/engine/planner/LogicalPlan.java | 197 +----------------- .../planner/nameresolver/NameResolver.java | 119 +++++++++-- .../nameresolver/ResolverByLegacy.java | 126 +++++++++++ 4 files changed, 236 insertions(+), 211 deletions(-) create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java index a38b29c4b9..36efba648f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java @@ -23,6 +23,8 @@ import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.engine.exception.NoSuchColumnException; +import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; +import org.apache.tajo.engine.planner.nameresolver.NameResolver; import java.util.ArrayList; import java.util.List; @@ -340,7 +342,8 @@ public Expr visitColumnReference(ExprNormalizedResult ctx, Stack stack, Co if (!(expr.hasQualifier() && CatalogUtil.isFQTableName(expr.getQualifier()))) { if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName()) && expr.getType() == OpType.Column) { try { - String normalized = ctx.plan.getNormalizedColumnName(ctx.block, expr); + String normalized = + NameResolver.resolve(ctx.plan, ctx.block, expr, NameResolveLevel.GLOBAL).getQualifiedName(); expr.setName(normalized); } catch (NoSuchColumnException nsc) { } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java index 6e923dd658..dede57acc5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java @@ -22,7 +22,6 @@ import org.apache.commons.lang.ObjectUtils; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.NotThreadSafe; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.engine.eval.EvalNode; @@ -30,9 +29,9 @@ import org.apache.tajo.engine.planner.graph.DirectedGraphCursor; import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph; import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.planner.nameresolver.NameResolver; import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; -import org.apache.tajo.util.Pair; +import org.apache.tajo.engine.planner.nameresolver.NameResolver; +import org.apache.tajo.engine.planner.nameresolver.ResolverByLegacy; import org.apache.tajo.util.TUtil; import java.lang.reflect.Constructor; @@ -263,198 +262,8 @@ public SimpleDirectedGraph getQueryBlockGraph() { return queryBlockGraph; } - public Pair normalizeQualifierAndCanonicalName(QueryBlock block, ColumnReferenceExpr columnRef) - throws PlanningException { - String qualifier; - String canonicalName; - - if (CatalogUtil.isFQTableName(columnRef.getQualifier())) { - qualifier = columnRef.getQualifier(); - canonicalName = columnRef.getCanonicalName(); - } else { - String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier()); - if (resolvedDatabaseName == null) { - throw new NoSuchColumnException(columnRef.getQualifier()); - } - qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier()); - canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); - } - - return new Pair(qualifier, canonicalName); - } - - public String getNormalizedColumnName(QueryBlock block, ColumnReferenceExpr columnRef) - throws PlanningException { - Column found = resolveColumn(block, columnRef); - if (found == null) { - throw new NoSuchColumnException(columnRef.getCanonicalName()); - } - return found.getQualifiedName(); - } - - public String resolveDatabase(QueryBlock block, String tableName) throws PlanningException { - List found = new ArrayList(); - for (RelationNode relation : block.getRelations()) { - // check alias name or table name - if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) || - CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) { - // obtain the database name - found.add(CatalogUtil.extractQualifier(relation.getTableName())); - } - } - - if (found.size() == 0) { - return null; - } else if (found.size() > 1) { - throw new PlanningException("Ambiguous table name \"" + tableName + "\""); - } - - return found.get(0); - } - - /** - * It resolves a column. - */ public Column resolveColumn(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { - if (columnRef.hasQualifier()) { - return resolveColumnWithQualifier(block, columnRef); - } else { - return resolveColumnWithoutQualifier(block, columnRef); - } - } - - private Column resolveColumnWithQualifier(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { - final String qualifier; - String canonicalName; - final String qualifiedName; - - Pair normalized = normalizeQualifierAndCanonicalName(block, columnRef); - qualifier = normalized.getFirst(); - canonicalName = normalized.getSecond(); - - qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); - - // Column column = resolveColumnFromRelationWithinBlock(block, qualifier, canonicalName);; - Column column = NameResolver.resolve(this, block, columnRef, NameResolveLevel.RELS_ONLY); - if (column == null) { - throw new NoSuchColumnException(canonicalName); - } - - // If code reach here, a column is found. - // But, it may be aliased from bottom logical node. - // If the column is aliased, the found name may not be used in upper node. - - // Here, we try to check if column reference is already aliased. - // If so, it replaces the name with aliased name. - LogicalNode currentNode = block.getCurrentNode(); - - // The condition (currentNode.getInSchema().contains(column)) means - // the column can be used at the current node. So, we don't need to find aliase name. - Schema currentNodeSchema = null; - if (currentNode != null) { - if (currentNode instanceof RelationNode) { - currentNodeSchema = ((RelationNode) currentNode).getTableSchema(); - } else { - currentNodeSchema = currentNode.getInSchema(); - } - } - - if (currentNode != null && !currentNodeSchema.contains(column) - && currentNode.getType() != NodeType.TABLE_SUBQUERY) { - List candidates = TUtil.newList(); - if (block.namedExprsMgr.isAliased(qualifiedName)) { - String alias = block.namedExprsMgr.getAlias(canonicalName); - Column found = resolveColumn(block, new ColumnReferenceExpr(alias)); - if (found != null) { - candidates.add(found); - } - } - if (!candidates.isEmpty()) { - return NameResolver.ensureUniqueColumn(candidates); - } - } - - return column; - } - - private Column resolveColumnWithoutQualifier(QueryBlock block, - ColumnReferenceExpr columnRef)throws PlanningException { - - List candidates = TUtil.newList(); - - // It tries to find a full qualified column name from all relations in the current block. - for (RelationNode rel : block.getRelations()) { - Column found = rel.getTableSchema().getColumn(columnRef.getName()); - if (found != null) { - candidates.add(found); - } - } - - if (!candidates.isEmpty()) { - return NameResolver.ensureUniqueColumn(candidates); - } - - // Trying to find the column within the current block - if (block.currentNode != null && block.currentNode.getInSchema() != null) { - Column found = block.currentNode.getInSchema().getColumn(columnRef.getCanonicalName()); - if (found != null) { - return found; - } - } - - if (block.getLatestNode() != null) { - Column found = block.getLatestNode().getOutSchema().getColumn(columnRef.getName()); - if (found != null) { - return found; - } - } - - - // Trying to find columns from aliased references. - if (block.namedExprsMgr.isAliased(columnRef.getCanonicalName())) { - String originalName = block.namedExprsMgr.getAlias(columnRef.getCanonicalName()); - Column found = resolveColumn(block, new ColumnReferenceExpr(originalName)); - if (found != null) { - candidates.add(found); - } - } - if (!candidates.isEmpty()) { - return NameResolver.ensureUniqueColumn(candidates); - } - - // This is an exception case. It means that there are some bugs in other parts. - LogicalNode blockRootNode = block.getRoot(); - if (blockRootNode != null && blockRootNode.getOutSchema().getColumn(columnRef.getCanonicalName()) != null) { - throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName()); - } - - // Trying to find columns from other relations in other blocks - for (QueryBlock eachBlock : queryBlocks.values()) { - for (RelationNode rel : eachBlock.getRelations()) { - Column found = rel.getTableSchema().getColumn(columnRef.getName()); - if (found != null) { - candidates.add(found); - } - } - } - - if (!candidates.isEmpty()) { - return NameResolver.ensureUniqueColumn(candidates); - } - - // Trying to find columns from schema in current block. - if (block.getSchema() != null) { - Column found = block.getSchema().getColumn(columnRef.getName()); - if (found != null) { - candidates.add(found); - } - } - - if (!candidates.isEmpty()) { - return NameResolver.ensureUniqueColumn(candidates); - } - - throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName()); + return NameResolver.resolve(this, block, columnRef, NameResolveLevel.GLOBAL); } public String getQueryGraphAsString() { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java index 3c0f729df9..a1d28a8568 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner.nameresolver; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.tajo.algebra.ColumnReferenceExpr; import org.apache.tajo.catalog.CatalogUtil; @@ -32,6 +33,7 @@ import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -43,6 +45,7 @@ public abstract class NameResolver { resolverMap.put(NameResolveLevel.RELS_ONLY, new ResolverByRels()); resolverMap.put(NameResolveLevel.RELS_AND_SUBEXPRS, new ResolverByRelsAndSubExprs()); resolverMap.put(NameResolveLevel.SUBEXPRS_AND_RELS, new ResolverBySubExprsAndRels()); + resolverMap.put(NameResolveLevel.GLOBAL, new ResolverByLegacy()); } public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column, @@ -90,7 +93,7 @@ public static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.Qu String canonicalName; if (columnRef.hasQualifier()) { - Pair normalized = plan.normalizeQualifierAndCanonicalName(block, columnRef); + Pair normalized = normalizeQualifierAndCanonicalName(plan, block, columnRef); qualifier = normalized.getFirst(); canonicalName = normalized.getSecond(); @@ -116,21 +119,7 @@ public static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.Qu return column; } else { - List candidates = TUtil.newList(); - - // It tries to find a full qualified column name from all relations in the current block. - for (RelationNode rel : block.getRelations()) { - Column found = rel.getTableSchema().getColumn(columnRef.getName()); - if (found != null) { - candidates.add(found); - } - } - - if (!candidates.isEmpty()) { - return ensureUniqueColumn(candidates); - } else { - return null; - } + return resolveUnqualifiedFromAllRelsInBlock(plan, block, columnRef); } } @@ -151,6 +140,104 @@ public static Column resolveSubExprReferences(LogicalPlan plan, LogicalPlan.Quer return null; } + public static Column resolveUnqualifiedFromAllRelsInBlock(LogicalPlan plan, LogicalPlan.QueryBlock block, + ColumnReferenceExpr column) throws VerifyException { + List candidates = TUtil.newList(); + + // It tries to find a full qualified column name from all relations in the current block. + for (RelationNode rel : block.getRelations()) { + Column found = rel.getTableSchema().getColumn(column.getName()); + if (found != null) { + candidates.add(found); + } + } + + if (!candidates.isEmpty()) { + return ensureUniqueColumn(candidates); + } else { + return null; + } + } + + public static Column resolveFromAllRelsInAllBlocks(LogicalPlan plan, LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef) throws VerifyException { + List candidates = Lists.newArrayList(); + // Trying to find columns from other relations in other blocks + for (LogicalPlan.QueryBlock eachBlock : plan.getQueryBlocks()) { + for (RelationNode rel : eachBlock.getRelations()) { + Column found = rel.getTableSchema().getColumn(columnRef.getName()); + if (found != null) { + candidates.add(found); + } + } + } + + if (!candidates.isEmpty()) { + return NameResolver.ensureUniqueColumn(candidates); + } else { + return null; + } + } + + public static Column resolveAliasedName(LogicalPlan plan, LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef) throws VerifyException { + List candidates = Lists.newArrayList(); + // Trying to find columns from schema in current block. + if (block.getSchema() != null) { + Column found = block.getSchema().getColumn(columnRef.getName()); + if (found != null) { + candidates.add(found); + } + } + + if (!candidates.isEmpty()) { + return NameResolver.ensureUniqueColumn(candidates); + } else { + return null; + } + } + + public static String resolveDatabase(LogicalPlan.QueryBlock block, String tableName) throws PlanningException { + List found = new ArrayList(); + for (RelationNode relation : block.getRelations()) { + // check alias name or table name + if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) || + CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) { + // obtain the database name + found.add(CatalogUtil.extractQualifier(relation.getTableName())); + } + } + + if (found.size() == 0) { + return null; + } else if (found.size() > 1) { + throw new PlanningException("Ambiguous table name \"" + tableName + "\""); + } + + return found.get(0); + } + + public static Pair normalizeQualifierAndCanonicalName(LogicalPlan plan, LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef) + throws PlanningException { + String qualifier; + String canonicalName; + + if (CatalogUtil.isFQTableName(columnRef.getQualifier())) { + qualifier = columnRef.getQualifier(); + canonicalName = columnRef.getCanonicalName(); + } else { + String resolvedDatabaseName = resolveDatabase(block, columnRef.getQualifier()); + if (resolvedDatabaseName == null) { + throw new NoSuchColumnException(columnRef.getQualifier()); + } + qualifier = CatalogUtil.buildFQName(resolvedDatabaseName, columnRef.getQualifier()); + canonicalName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); + } + + return new Pair(qualifier, canonicalName); + } + abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java new file mode 100644 index 0000000000..45b89b5e3d --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.nameresolver; + +import org.apache.tajo.algebra.ColumnReferenceExpr; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.engine.exception.NoSuchColumnException; +import org.apache.tajo.engine.planner.LogicalPlan; +import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.planner.logical.RelationNode; +import org.apache.tajo.util.Pair; +import org.apache.tajo.util.TUtil; + +import java.util.List; + +public class ResolverByLegacy extends NameResolver { + @Override + public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) + throws PlanningException { + + if (columnRef.hasQualifier()) { + return resolveColumnWithQualifier(plan, block, columnRef); + } else { + return resolveColumnWithoutQualifier(plan, block, columnRef); + } + } + + private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) + throws PlanningException { + final String qualifier; + String canonicalName; + final String qualifiedName; + + Pair normalized = normalizeQualifierAndCanonicalName(plan, block, columnRef); + qualifier = normalized.getFirst(); + canonicalName = normalized.getSecond(); + qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); + + Column found = resolveFromRelsWithinBlock(plan, block, columnRef); + if (found == null) { + throw new NoSuchColumnException(columnRef.getCanonicalName()); + } + + // If code reach here, a column is found. + // But, it may be aliased from bottom logical node. + // If the column is aliased, the found name may not be used in upper node. + + // Here, we try to check if column reference is already aliased. + // If so, it replaces the name with aliased name. + LogicalNode currentNode = block.getCurrentNode(); + + // The condition (currentNode.getInSchema().contains(column)) means + // the column can be used at the current node. So, we don't need to find aliase name. + Schema currentNodeSchema = null; + if (currentNode != null) { + if (currentNode instanceof RelationNode) { + currentNodeSchema = ((RelationNode) currentNode).getTableSchema(); + } else { + currentNodeSchema = currentNode.getInSchema(); + } + } + + if (currentNode != null && !currentNodeSchema.contains(found) + && currentNode.getType() != NodeType.TABLE_SUBQUERY) { + List candidates = TUtil.newList(); + if (block.getNamedExprsManager().isAliased(qualifiedName)) { + String alias = block.getNamedExprsManager().getAlias(canonicalName); + found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolveLevel.GLOBAL); + if (found != null) { + candidates.add(found); + } + } + if (!candidates.isEmpty()) { + return ensureUniqueColumn(candidates); + } + } + + return found; + } + + static Column resolveColumnWithoutQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef)throws PlanningException { + + Column found = resolveUnqualifiedFromAllRelsInBlock(plan, block, columnRef); + if (found != null) { + return found; + } + + found = resolveAliasedName(plan, block, columnRef); + if (found != null) { + return found; + } + + found = resolveSubExprReferences(plan, block, columnRef); + if (found != null) { + return found; + } + + found = resolveFromAllRelsInAllBlocks(plan, block, columnRef); + if (found != null) { + return found; + } + + throw new NoSuchColumnException("ERROR: no such a column name "+ columnRef.getCanonicalName()); + } +} From ce12386d5e188679f48042ddc4262de59f9633be Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 14 Jul 2014 13:53:42 +0900 Subject: [PATCH 05/10] add comments. --- .../tajo/engine/planner/ExprAnnotator.java | 2 +- .../tajo/engine/planner/ExprNormalizer.java | 4 +- .../tajo/engine/planner/LogicalPlan.java | 11 +- .../tajo/engine/planner/LogicalPlanner.java | 6 +- .../nameresolver/NameResolveLevel.java | 8 +- .../planner/nameresolver/NameResolver.java | 183 +++++++++++------- .../nameresolver/ResolverByLegacy.java | 16 +- .../ResolverByRelsAndSubExprs.java | 2 +- .../ResolverBySubExprsAndRels.java | 2 +- .../tajo/engine/query/TestSelectQuery.java | 8 + 10 files changed, 146 insertions(+), 96 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java index d958e027f3..fde5e12b82 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java @@ -548,7 +548,7 @@ public EvalNode visitColumnReference(Context ctx, Stack stack, ColumnRefer Column column; switch (ctx.columnRsvLevel) { - case GLOBAL: + case LEGACY: column = ctx.plan.resolveColumn(ctx.currentBlock, expr); break; case RELS_ONLY: diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java index 36efba648f..735d0820d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java @@ -18,7 +18,6 @@ package org.apache.tajo.engine.planner; -import com.google.common.collect.Sets; import com.google.common.collect.Sets; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogUtil; @@ -29,7 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; -import java.util.Set; import java.util.Stack; /** @@ -343,7 +341,7 @@ public Expr visitColumnReference(ExprNormalizedResult ctx, Stack stack, Co if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName()) && expr.getType() == OpType.Column) { try { String normalized = - NameResolver.resolve(ctx.plan, ctx.block, expr, NameResolveLevel.GLOBAL).getQualifiedName(); + NameResolver.resolve(ctx.plan, ctx.block, expr, NameResolveLevel.LEGACY).getQualifiedName(); expr.setName(normalized); } catch (NoSuchColumnException nsc) { } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java index dede57acc5..c83899bc7a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java @@ -25,13 +25,14 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.exception.NoSuchColumnException; import org.apache.tajo.engine.planner.graph.DirectedGraphCursor; import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph; -import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.engine.planner.logical.LogicalRootNode; +import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.planner.logical.RelationNode; import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; import org.apache.tajo.engine.planner.nameresolver.NameResolver; -import org.apache.tajo.engine.planner.nameresolver.ResolverByLegacy; import org.apache.tajo.util.TUtil; import java.lang.reflect.Constructor; @@ -45,7 +46,6 @@ public class LogicalPlan { /** the prefix character for virtual tables */ public static final char VIRTUAL_TABLE_PREFIX='#'; public static final char NONAMED_COLUMN_PREFIX='?'; - public static final char NONAMED_WINDOW_PREFIX='^'; /** it indicates the root block */ public static final String ROOT_BLOCK = VIRTUAL_TABLE_PREFIX + "ROOT"; @@ -54,7 +54,6 @@ public class LogicalPlan { private int nextPid = 0; private Integer noNameBlockId = 0; private Integer noNameColumnId = 0; - private Integer noNameWindowId = 0; /** a map from between a block name to a block plan */ private Map queryBlocks = new LinkedHashMap(); @@ -263,7 +262,7 @@ public SimpleDirectedGraph getQueryBlockGraph() { } public Column resolveColumn(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { - return NameResolver.resolve(this, block, columnRef, NameResolveLevel.GLOBAL); + return NameResolver.resolve(this, block, columnRef, NameResolveLevel.LEGACY); } public String getQueryGraphAsString() { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 9334a7364c..d0a62cd624 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -1053,7 +1053,7 @@ public LogicalNode visitJoin(PlanContext context, Stack stack, Join join) EvalNode joinCondition = null; if (join.hasQual()) { EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, join.getQual(), - NameResolveLevel.GLOBAL); + NameResolveLevel.LEGACY); joinCondition = AlgebraicUtil.eliminateConstantExprs(evalNode); } @@ -1083,7 +1083,7 @@ private List getNewlyEvaluatedExprsForJoin(LogicalPlan plan, QueryBlock for (Iterator it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr namedExpr = it.next(); try { - evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolveLevel.GLOBAL); + evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolveLevel.LEGACY); if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, evalNode, joinNode, stack.peek().getType() != OpType.Join)) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); newlyEvaluatedExprs.add(namedExpr.getAlias()); @@ -1153,7 +1153,7 @@ private LogicalNode createCartesianProduct(PlanContext context, LogicalNode left for (Iterator it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr namedExpr = it.next(); try { - evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolveLevel.GLOBAL); + evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolveLevel.LEGACY); if (EvalTreeUtil.findDistinctAggFunction(evalNode).size() == 0) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); newlyEvaluatedExprs.add(namedExpr.getAlias()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java index caaedd06d0..148d2c97aa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java @@ -19,8 +19,8 @@ package org.apache.tajo.engine.planner.nameresolver; public enum NameResolveLevel { - RELS_ONLY, // finds from only relations - RELS_AND_SUBEXPRS, // finds from only relations and subexprs - SUBEXPRS_AND_RELS, - GLOBAL + RELS_ONLY, // finding from only relations + RELS_AND_SUBEXPRS, // finding from relations and subexprs in a place + SUBEXPRS_AND_RELS, // finding from subexprs and relations in a place + LEGACY // Finding in a legacy manner (globally) } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java index a1d28a8568..33b7f00a47 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java @@ -45,55 +45,66 @@ public abstract class NameResolver { resolverMap.put(NameResolveLevel.RELS_ONLY, new ResolverByRels()); resolverMap.put(NameResolveLevel.RELS_AND_SUBEXPRS, new ResolverByRelsAndSubExprs()); resolverMap.put(NameResolveLevel.SUBEXPRS_AND_RELS, new ResolverBySubExprsAndRels()); - resolverMap.put(NameResolveLevel.GLOBAL, new ResolverByLegacy()); + resolverMap.put(NameResolveLevel.LEGACY, new ResolverByLegacy()); + } + + abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) + throws PlanningException; + + /** + * Try to find the database name + * + * @param block the current block + * @param tableName The table name + * @return The found database name + * @throws PlanningException + */ + public static String resolveDatabase(LogicalPlan.QueryBlock block, String tableName) throws PlanningException { + List found = new ArrayList(); + for (RelationNode relation : block.getRelations()) { + // check alias name or table name + if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) || + CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) { + // obtain the database name + found.add(CatalogUtil.extractQualifier(relation.getTableName())); + } + } + + if (found.size() == 0) { + return null; + } else if (found.size() > 1) { + throw new PlanningException("Ambiguous table name \"" + tableName + "\""); + } + + return found.get(0); } public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column, NameResolveLevel level) throws PlanningException { if (!resolverMap.containsKey(level)) { - throw new PlanningException("Unsupported name resolving level"); + throw new PlanningException("Unsupported name resolving level: " + level.name()); } return resolverMap.get(level).resolve(plan, block, column); } - public static Column ensureUniqueColumn(List candidates) - throws VerifyException { - if (candidates.size() == 1) { - return candidates.get(0); - } else if (candidates.size() > 2) { - StringBuilder sb = new StringBuilder(); - boolean first = true; - for (Column column : candidates) { - if (first) { - first = false; - } else { - sb.append(", "); - } - sb.append(column); - } - throw new AmbiguousFieldException("Ambiguous Column Name: " + sb.toString()); - } else { - return null; - } - } - /** - * It tries to resolve the variable name from relations within a given query block. - * If a variable name is qualified, it tries to resolve the name from the relation corresponding to the qualifier. + * Try to find a column from all relations within a given query block. + * If a given column reference is qualified, it tries to resolve the name + * from only the relation corresponding to the qualifier. * - * @param plan - * @param block - * @param columnRef - * @return + * @param plan The logical plan + * @param block The current query block + * @param columnRef The column reference to be found + * @return The found column * @throws PlanningException */ - public static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.QueryBlock block, + static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { String qualifier; String canonicalName; if (columnRef.hasQualifier()) { - Pair normalized = normalizeQualifierAndCanonicalName(plan, block, columnRef); + Pair normalized = normalizeQualifierAndCanonicalName(block, columnRef); qualifier = normalized.getFirst(); canonicalName = normalized.getSecond(); @@ -119,13 +130,20 @@ public static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.Qu return column; } else { - return resolveUnqualifiedFromAllRelsInBlock(plan, block, columnRef); + return resolveFromAllRelsInBlock(block, columnRef); } } - public static Column resolveSubExprReferences(LogicalPlan plan, LogicalPlan.QueryBlock block, - ColumnReferenceExpr columnRef) throws NoSuchColumnException { - // Trying to find the column within the current block + /** + * Try to find the column from the current node and child node. It can find subexprs generated from the optimizer. + * + * @param block The current query block + * @param columnRef The column reference to be found + * @return The found column + */ + static Column resolveFromCurrentAndChildNode(LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) + throws NoSuchColumnException { + if (block.getCurrentNode() != null && block.getCurrentNode().getInSchema() != null) { Column found = block.getCurrentNode().getInSchema().getColumn(columnRef.getCanonicalName()); if (found != null) { @@ -140,13 +158,19 @@ public static Column resolveSubExprReferences(LogicalPlan plan, LogicalPlan.Quer return null; } - public static Column resolveUnqualifiedFromAllRelsInBlock(LogicalPlan plan, LogicalPlan.QueryBlock block, - ColumnReferenceExpr column) throws VerifyException { + /** + * It tries to find a full qualified column name from all relations in the current block. + * + * @param block The current query block + * @param columnRef The column reference to be found + * @return The found column + */ + static Column resolveFromAllRelsInBlock(LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef) throws VerifyException { List candidates = TUtil.newList(); - // It tries to find a full qualified column name from all relations in the current block. for (RelationNode rel : block.getRelations()) { - Column found = rel.getTableSchema().getColumn(column.getName()); + Column found = rel.getTableSchema().getColumn(columnRef.getName()); if (found != null) { candidates.add(found); } @@ -159,11 +183,20 @@ public static Column resolveUnqualifiedFromAllRelsInBlock(LogicalPlan plan, Logi } } - public static Column resolveFromAllRelsInAllBlocks(LogicalPlan plan, LogicalPlan.QueryBlock block, - ColumnReferenceExpr columnRef) throws VerifyException { + /** + * Trying to find a column from all relations in other blocks + * + * @param plan The logical plan + * @param columnRef The column reference to be found + * @return The found column + */ + static Column resolveFromAllRelsInAllBlocks(LogicalPlan plan, ColumnReferenceExpr columnRef) throws VerifyException { + List candidates = Lists.newArrayList(); - // Trying to find columns from other relations in other blocks + + // from all relations of all query blocks for (LogicalPlan.QueryBlock eachBlock : plan.getQueryBlocks()) { + for (RelationNode rel : eachBlock.getRelations()) { Column found = rel.getTableSchema().getColumn(columnRef.getName()); if (found != null) { @@ -179,10 +212,16 @@ public static Column resolveFromAllRelsInAllBlocks(LogicalPlan plan, LogicalPlan } } - public static Column resolveAliasedName(LogicalPlan plan, LogicalPlan.QueryBlock block, - ColumnReferenceExpr columnRef) throws VerifyException { + /** + * Try to find a column from the final schema of the current block. + * + * @param block The current query block + * @param columnRef The column reference to be found + * @return The found column + */ + static Column resolveAliasedName(LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) throws VerifyException { List candidates = Lists.newArrayList(); - // Trying to find columns from schema in current block. + if (block.getSchema() != null) { Column found = block.getSchema().getColumn(columnRef.getName()); if (found != null) { @@ -197,28 +236,17 @@ public static Column resolveAliasedName(LogicalPlan plan, LogicalPlan.QueryBlock } } - public static String resolveDatabase(LogicalPlan.QueryBlock block, String tableName) throws PlanningException { - List found = new ArrayList(); - for (RelationNode relation : block.getRelations()) { - // check alias name or table name - if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) || - CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) { - // obtain the database name - found.add(CatalogUtil.extractQualifier(relation.getTableName())); - } - } - - if (found.size() == 0) { - return null; - } else if (found.size() > 1) { - throw new PlanningException("Ambiguous table name \"" + tableName + "\""); - } - - return found.get(0); - } - - public static Pair normalizeQualifierAndCanonicalName(LogicalPlan plan, LogicalPlan.QueryBlock block, - ColumnReferenceExpr columnRef) + /** + * It returns a pair of names, which the first value is ${database}.${table} and the second value + * is a simple column name. + * + * @param block The current block + * @param columnRef The column name + * @return A pair of normalized qualifier and column name + * @throws PlanningException + */ + static Pair normalizeQualifierAndCanonicalName(LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef) throws PlanningException { String qualifier; String canonicalName; @@ -238,6 +266,23 @@ public static Pair normalizeQualifierAndCanonicalName(LogicalPla return new Pair(qualifier, canonicalName); } - abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) - throws PlanningException; + static Column ensureUniqueColumn(List candidates) throws VerifyException { + if (candidates.size() == 1) { + return candidates.get(0); + } else if (candidates.size() > 2) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Column column : candidates) { + if (first) { + first = false; + } else { + sb.append(", "); + } + sb.append(column); + } + throw new AmbiguousFieldException("Ambiguous Column Name: " + sb.toString()); + } else { + return null; + } + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java index 45b89b5e3d..10d0f9390e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java @@ -45,13 +45,13 @@ public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnRefe } } - private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) - throws PlanningException { + private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block, + ColumnReferenceExpr columnRef) throws PlanningException { final String qualifier; String canonicalName; final String qualifiedName; - Pair normalized = normalizeQualifierAndCanonicalName(plan, block, columnRef); + Pair normalized = normalizeQualifierAndCanonicalName(block, columnRef); qualifier = normalized.getFirst(); canonicalName = normalized.getSecond(); qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); @@ -85,7 +85,7 @@ private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.Q List candidates = TUtil.newList(); if (block.getNamedExprsManager().isAliased(qualifiedName)) { String alias = block.getNamedExprsManager().getAlias(canonicalName); - found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolveLevel.GLOBAL); + found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolveLevel.LEGACY); if (found != null) { candidates.add(found); } @@ -101,22 +101,22 @@ private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.Q static Column resolveColumnWithoutQualifier(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef)throws PlanningException { - Column found = resolveUnqualifiedFromAllRelsInBlock(plan, block, columnRef); + Column found = resolveFromAllRelsInBlock(block, columnRef); if (found != null) { return found; } - found = resolveAliasedName(plan, block, columnRef); + found = resolveAliasedName(block, columnRef); if (found != null) { return found; } - found = resolveSubExprReferences(plan, block, columnRef); + found = resolveFromCurrentAndChildNode(block, columnRef); if (found != null) { return found; } - found = resolveFromAllRelsInAllBlocks(plan, block, columnRef); + found = resolveFromAllRelsInAllBlocks(plan, columnRef); if (found != null) { return found; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java index 4009c2e199..7ca3c535d3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java @@ -31,7 +31,7 @@ public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnRefe Column column = resolveFromRelsWithinBlock(plan, block, columnRef); if (column == null) { - column = resolveSubExprReferences(plan, block, columnRef); + column = resolveFromCurrentAndChildNode(block, columnRef); } if (column == null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java index 6908a7810d..7337ecee5e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java @@ -29,7 +29,7 @@ public class ResolverBySubExprsAndRels extends NameResolver { public Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { - Column column = resolveSubExprReferences(plan, block, columnRef); + Column column = resolveFromCurrentAndChildNode(block, columnRef); if (column == null) { column = resolveFromRelsWithinBlock(plan, block, columnRef); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 2b0f4af0bf..a4e00f8848 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -120,6 +120,14 @@ public final void testSelectColumnAlias1() throws Exception { cleanupQuery(res); } + @Test + public final void testSelectColumnAliasExistingInRelation1() throws Exception { + // select (l_orderkey + l_orderkey) l_orderkey as col2 from lineitem where l_orderkey > 2; + ResultSet res = executeString("select (l_orderkey + l_orderkey) l_orderkey as col2 from lineitem where l_orderkey > 2"); + System.out.println(resultSetToString(res)); + cleanupQuery(res); + } + @Test public final void testSelectSameConstantsWithDifferentAliases() throws Exception { // select l_orderkey, '20130819' as date1, '20130819' as date2 from lineitem where l_orderkey > -1; From 0cac367998ce0399d0e31e6b5669ba74e1bb91a8 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 14 Jul 2014 14:09:37 +0900 Subject: [PATCH 06/10] added more unit tests. --- .../tajo/engine/query/TestSelectQuery.java | 17 ++++++++++++++--- ...testSelectColumnAliasExistingInRelation1.sql | 1 + ...testSelectColumnAliasExistingInRelation2.sql | 1 + ...tSelectColumnAliasExistingInRelation1.result | 4 ++++ ...tSelectColumnAliasExistingInRelation2.result | 7 +++++++ 5 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation1.sql create mode 100644 tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation2.sql create mode 100644 tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation1.result create mode 100644 tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation2.result diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index a4e00f8848..8898067fe4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -122,12 +122,23 @@ public final void testSelectColumnAlias1() throws Exception { @Test public final void testSelectColumnAliasExistingInRelation1() throws Exception { - // select (l_orderkey + l_orderkey) l_orderkey as col2 from lineitem where l_orderkey > 2; - ResultSet res = executeString("select (l_orderkey + l_orderkey) l_orderkey as col2 from lineitem where l_orderkey > 2"); - System.out.println(resultSetToString(res)); + // We intend that 'l_orderkey' in where clause points to "default.lineitem.l_orderkey" + // select (l_orderkey + l_orderkey) l_orderkey from lineitem where l_orderkey > 2; + ResultSet res = executeQuery(); + assertResultSet(res); cleanupQuery(res); } + @Test + public final void testSelectColumnAliasExistingInRelation2() throws Exception { + // We intend that 'l_orderkey' in orderby clause points to (-l_orderkey). + // select (-l_orderkey) as l_orderkey from lineitem order by l_orderkey; + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test public final void testSelectSameConstantsWithDifferentAliases() throws Exception { // select l_orderkey, '20130819' as date1, '20130819' as date2 from lineitem where l_orderkey > -1; diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation1.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation1.sql new file mode 100644 index 0000000000..91170e3468 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation1.sql @@ -0,0 +1 @@ +select (l_orderkey + l_orderkey) l_orderkey from lineitem where l_orderkey > 2; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation2.sql new file mode 100644 index 0000000000..89f63fd772 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testSelectColumnAliasExistingInRelation2.sql @@ -0,0 +1 @@ +select (-l_orderkey) as l_orderkey from lineitem order by l_orderkey; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation1.result b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation1.result new file mode 100644 index 0000000000..55e2b42174 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation1.result @@ -0,0 +1,4 @@ +l_orderkey +------------------------------- +6 +6 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation2.result b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation2.result new file mode 100644 index 0000000000..f0cf700625 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectColumnAliasExistingInRelation2.result @@ -0,0 +1,7 @@ +l_orderkey +------------------------------- +-3 +-3 +-2 +-1 +-1 \ No newline at end of file From d51d950a2b5db0a482a0589b0c3b14cb0f3ddafd Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 14 Jul 2014 20:51:22 +0900 Subject: [PATCH 07/10] Reflected the review comments and add some comments. --- .../planner/LogicalPlanPreprocessor.java | 5 ++-- .../tajo/engine/planner/LogicalPlanner.java | 3 --- .../planner/TestLogicalPlanPreprocessor.java | 23 ------------------- 3 files changed, 3 insertions(+), 28 deletions(-) delete mode 100644 tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanPreprocessor.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index cec1738d6b..7e5d423031 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -198,11 +198,12 @@ public LogicalNode visitProjection(PreprocessContext ctx, Stack stack, Pro expr.setNamedExprs(rewrittenTargets.toArray(new NamedExpr[rewrittenTargets.size()])); } + // 1) Normalize field names into full qualified names + // 2) Register explicit column aliases to block NamedExpr[] projectTargetExprs = expr.getNamedExprs(); NameRefInSelectListNormalizer normalizer = new NameRefInSelectListNormalizer(); - for (int i = 0; i < expr.getNamedExprs().length; i++) { - NamedExpr namedExpr = expr.getNamedExprs()[i]; + NamedExpr namedExpr = projectTargetExprs[i]; normalizer.visit(ctx, new Stack(), namedExpr.getExpr()); if (namedExpr.getExpr().getType() == OpType.Column && namedExpr.hasAlias()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index d0a62cd624..1973b40bed 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -1204,9 +1204,6 @@ public ScanNode visitRelation(PlanContext context, Stack stack, Relation e Set newlyEvaluatedExprsReferences = new LinkedHashSet(); for (Iterator iterator = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); iterator.hasNext();) { NamedExpr rawTarget = iterator.next(); -// if (rawTarget.getExpr().getType() == OpType.Column) { -// continue; -// } try { EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(), NameResolveLevel.RELS_ONLY); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanPreprocessor.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanPreprocessor.java deleted file mode 100644 index e74fce2a2f..0000000000 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanPreprocessor.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.planner; - - -public class TestLogicalPlanPreprocessor { -} From 03f7921c82b8c925af05e9d8fc888042359638d4 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 14 Jul 2014 23:35:49 +0900 Subject: [PATCH 08/10] Added more explanation and renamed level to mode. --- .../tajo/engine/planner/ExprAnnotator.java | 8 +- .../tajo/engine/planner/ExprNormalizer.java | 4 +- .../tajo/engine/planner/LogicalPlan.java | 4 +- .../planner/LogicalPlanPreprocessor.java | 6 +- .../tajo/engine/planner/LogicalPlanner.java | 34 ++++---- .../nameresolver/NameResolveLevel.java | 26 ------ .../planner/nameresolver/NameResolver.java | 21 ++--- .../nameresolver/NameResolvingMode.java | 80 +++++++++++++++++++ .../nameresolver/ResolverByLegacy.java | 2 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 4 +- 10 files changed, 123 insertions(+), 66 deletions(-) delete mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java index fde5e12b82..39fd08acf5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java @@ -32,8 +32,8 @@ import org.apache.tajo.engine.function.AggFunction; import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.engine.planner.nameresolver.NameResolver; -import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; import org.apache.tajo.exception.InternalException; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; @@ -68,9 +68,9 @@ public ExprAnnotator(CatalogService catalog) { static class Context { LogicalPlan plan; LogicalPlan.QueryBlock currentBlock; - NameResolveLevel columnRsvLevel; + NameResolvingMode columnRsvLevel; - public Context(LogicalPlan plan, LogicalPlan.QueryBlock block, NameResolveLevel colRsvLevel) { + public Context(LogicalPlan plan, LogicalPlan.QueryBlock block, NameResolvingMode colRsvLevel) { this.plan = plan; this.currentBlock = block; this.columnRsvLevel = colRsvLevel; @@ -78,7 +78,7 @@ public Context(LogicalPlan plan, LogicalPlan.QueryBlock block, NameResolveLevel } public EvalNode createEvalNode(LogicalPlan plan, LogicalPlan.QueryBlock block, Expr expr, - NameResolveLevel colRsvLevel) + NameResolvingMode colRsvLevel) throws PlanningException { Context context = new Context(plan, block, colRsvLevel); return AlgebraicUtil.eliminateConstantExprs(visit(context, new Stack(), expr)); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java index 735d0820d7..5b61b74ec1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java @@ -22,7 +22,7 @@ import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.engine.exception.NoSuchColumnException; -import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; +import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.engine.planner.nameresolver.NameResolver; import java.util.ArrayList; @@ -341,7 +341,7 @@ public Expr visitColumnReference(ExprNormalizedResult ctx, Stack stack, Co if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName()) && expr.getType() == OpType.Column) { try { String normalized = - NameResolver.resolve(ctx.plan, ctx.block, expr, NameResolveLevel.LEGACY).getQualifiedName(); + NameResolver.resolve(ctx.plan, ctx.block, expr, NameResolvingMode.LEGACY).getQualifiedName(); expr.setName(normalized); } catch (NoSuchColumnException nsc) { } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java index c83899bc7a..86bacefbd5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java @@ -31,7 +31,7 @@ import org.apache.tajo.engine.planner.logical.LogicalRootNode; import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.engine.planner.logical.RelationNode; -import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; +import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.engine.planner.nameresolver.NameResolver; import org.apache.tajo.util.TUtil; @@ -262,7 +262,7 @@ public SimpleDirectedGraph getQueryBlockGraph() { } public Column resolveColumn(QueryBlock block, ColumnReferenceExpr columnRef) throws PlanningException { - return NameResolver.resolve(this, block, columnRef, NameResolveLevel.LEGACY); + return NameResolver.resolve(this, block, columnRef, NameResolvingMode.LEGACY); } public String getQueryGraphAsString() { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index 7e5d423031..b19d9ee041 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -26,8 +26,8 @@ import org.apache.tajo.engine.exception.NoSuchColumnException; import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.engine.planner.nameresolver.NameResolver; -import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.master.session.Session; import org.apache.tajo.util.TUtil; @@ -284,7 +284,7 @@ public LogicalNode visitGroupBy(PreprocessContext ctx, Stack stack, Aggreg for (int i = 0; i < finalTargetNum; i++) { NamedExpr namedExpr = projection.getNamedExprs()[i]; EvalNode evalNode = annotator.createEvalNode(ctx.plan, ctx.currentBlock, namedExpr.getExpr(), - NameResolveLevel.SUBEXPRS_AND_RELS); + NameResolvingMode.SUBEXPRS_AND_RELS); if (namedExpr.hasAlias()) { targets[i] = new Target(evalNode, namedExpr.getAlias()); @@ -473,7 +473,7 @@ public Expr visitColumnReference(PreprocessContext ctx, Stack stack, Colum throws PlanningException { String normalized = NameResolver.resolve(ctx.plan, ctx.currentBlock, expr, - NameResolveLevel.RELS_AND_SUBEXPRS).getQualifiedName(); + NameResolvingMode.RELS_AND_SUBEXPRS).getQualifiedName(); expr.setName(normalized); return expr; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 1973b40bed..34626a2c42 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -40,7 +40,7 @@ import org.apache.tajo.engine.exception.VerifyException; import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; +import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule; import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.master.session.Session; @@ -263,7 +263,7 @@ private void setRawTargets(PlanContext context, Target[] targets, String[] refer for (int i = 0; i < projection.getNamedExprs().length; i++) { NamedExpr namedExpr = projection.getNamedExprs()[i]; EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), - NameResolveLevel.SUBEXPRS_AND_RELS); + NameResolvingMode.SUBEXPRS_AND_RELS); rawTargets[i] = new Target(evalNode, referenceNames[i]); } // it's for debugging or unit testing @@ -386,7 +386,7 @@ private EvalExprNode buildPlanForNoneFromStatement(PlanContext context, Stack stack, Limit limit) LogicalNode child; if (limit.getFetchFirstNum().getType() == OpType.Literal) { firstFetNum = exprAnnotator.createEvalNode(context.plan, block, limit.getFetchFirstNum(), - NameResolveLevel.RELS_ONLY); + NameResolvingMode.RELS_ONLY); //////////////////////////////////////////////////////// // Visit and Build Child Plan @@ -747,7 +747,7 @@ public LimitNode visitLimit(PlanContext context, Stack stack, Limit limit) } else { NamedExpr namedExpr = block.namedExprsMgr.getNamedExpr(referName); firstFetNum = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr(), - NameResolveLevel.SUBEXPRS_AND_RELS); + NameResolvingMode.SUBEXPRS_AND_RELS); block.namedExprsMgr.markAsEvaluated(referName, firstFetNum); } } @@ -844,7 +844,7 @@ public LogicalNode visitHaving(PlanContext context, Stack stack, Having ex } else { NamedExpr namedExpr = block.namedExprsMgr.getNamedExpr(referName); havingCondition = exprAnnotator.createEvalNode(context.plan, block, namedExpr.getExpr(), - NameResolveLevel.SUBEXPRS_AND_RELS); + NameResolvingMode.SUBEXPRS_AND_RELS); block.namedExprsMgr.markAsEvaluated(referName, havingCondition); } @@ -912,7 +912,7 @@ public LogicalNode visitGroupBy(PlanContext context, Stack stack, Aggregat try { includeDistinctFunction |= PlannerUtil.existsDistinctAggregationFunction(namedExpr.getExpr()); EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, namedExpr.getExpr(), - NameResolveLevel.SUBEXPRS_AND_RELS); + NameResolvingMode.SUBEXPRS_AND_RELS); if (evalNode.getType() == EvalType.AGG_FUNCTION) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); aggEvalNames.add(namedExpr.getAlias()); @@ -999,7 +999,7 @@ public SelectionNode visitFilter(PlanContext context, Stack stack, Selecti // Create EvalNode for a search condition. EvalNode searchCondition = exprAnnotator.createEvalNode(context.plan, block, selection.getQual(), - NameResolveLevel.RELS_AND_SUBEXPRS); + NameResolvingMode.RELS_AND_SUBEXPRS); EvalNode simplified = AlgebraicUtil.eliminateConstantExprs(searchCondition); // set selection condition selectionNode.setQual(simplified); @@ -1053,7 +1053,7 @@ public LogicalNode visitJoin(PlanContext context, Stack stack, Join join) EvalNode joinCondition = null; if (join.hasQual()) { EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, join.getQual(), - NameResolveLevel.LEGACY); + NameResolvingMode.LEGACY); joinCondition = AlgebraicUtil.eliminateConstantExprs(evalNode); } @@ -1083,7 +1083,7 @@ private List getNewlyEvaluatedExprsForJoin(LogicalPlan plan, QueryBlock for (Iterator it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr namedExpr = it.next(); try { - evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolveLevel.LEGACY); + evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolvingMode.LEGACY); if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, evalNode, joinNode, stack.peek().getType() != OpType.Join)) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); newlyEvaluatedExprs.add(namedExpr.getAlias()); @@ -1153,7 +1153,7 @@ private LogicalNode createCartesianProduct(PlanContext context, LogicalNode left for (Iterator it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr namedExpr = it.next(); try { - evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolveLevel.LEGACY); + evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), NameResolvingMode.LEGACY); if (EvalTreeUtil.findDistinctAggFunction(evalNode).size() == 0) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); newlyEvaluatedExprs.add(namedExpr.getAlias()); @@ -1206,7 +1206,7 @@ public ScanNode visitRelation(PlanContext context, Stack stack, Relation e NamedExpr rawTarget = iterator.next(); try { EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(), - NameResolveLevel.RELS_ONLY); + NameResolvingMode.RELS_ONLY); if (checkIfBeEvaluatedAtRelation(block, evalNode, scanNode)) { block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode); newlyEvaluatedExprsReferences.add(rawTarget.getAlias()); // newly added exr @@ -1223,7 +1223,7 @@ public ScanNode visitRelation(PlanContext context, Stack stack, Relation e for (String reference : newlyEvaluatedExprsReferences) { NamedExpr refrer = block.namedExprsMgr.getNamedExpr(reference); EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, block, refrer.getExpr(), - NameResolveLevel.RELS_ONLY); + NameResolvingMode.RELS_ONLY); targets.add(new Target(evalNode, reference)); } @@ -1282,7 +1282,7 @@ public TableSubQueryNode visitTableSubQuery(PlanContext context, Stack sta for (NamedExpr rawTarget : block.namedExprsMgr.getAllNamedExprs()) { try { EvalNode evalNode = exprAnnotator.createEvalNode(context.plan, context.queryBlock, rawTarget.getExpr(), - NameResolveLevel.RELS_ONLY); + NameResolvingMode.RELS_ONLY); if (checkIfBeEvaluatedAtRelation(block, evalNode, subQueryNode)) { block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode); newlyEvaluatedExprs.add(rawTarget.getAlias()); // newly added exr diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java deleted file mode 100644 index 148d2c97aa..0000000000 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolveLevel.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.planner.nameresolver; - -public enum NameResolveLevel { - RELS_ONLY, // finding from only relations - RELS_AND_SUBEXPRS, // finding from relations and subexprs in a place - SUBEXPRS_AND_RELS, // finding from subexprs and relations in a place - LEGACY // Finding in a legacy manner (globally) -} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java index 33b7f00a47..aee5d43b6f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java @@ -37,15 +37,18 @@ import java.util.List; import java.util.Map; +/** + * NameResolver utility + */ public abstract class NameResolver { - public static Map resolverMap = Maps.newHashMap(); + public static Map resolverMap = Maps.newHashMap(); static { - resolverMap.put(NameResolveLevel.RELS_ONLY, new ResolverByRels()); - resolverMap.put(NameResolveLevel.RELS_AND_SUBEXPRS, new ResolverByRelsAndSubExprs()); - resolverMap.put(NameResolveLevel.SUBEXPRS_AND_RELS, new ResolverBySubExprsAndRels()); - resolverMap.put(NameResolveLevel.LEGACY, new ResolverByLegacy()); + resolverMap.put(NameResolvingMode.RELS_ONLY, new ResolverByRels()); + resolverMap.put(NameResolvingMode.RELS_AND_SUBEXPRS, new ResolverByRelsAndSubExprs()); + resolverMap.put(NameResolvingMode.SUBEXPRS_AND_RELS, new ResolverBySubExprsAndRels()); + resolverMap.put(NameResolvingMode.LEGACY, new ResolverByLegacy()); } abstract Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr columnRef) @@ -80,11 +83,11 @@ public static String resolveDatabase(LogicalPlan.QueryBlock block, String tableN } public static Column resolve(LogicalPlan plan, LogicalPlan.QueryBlock block, ColumnReferenceExpr column, - NameResolveLevel level) throws PlanningException { - if (!resolverMap.containsKey(level)) { - throw new PlanningException("Unsupported name resolving level: " + level.name()); + NameResolvingMode mode) throws PlanningException { + if (!resolverMap.containsKey(mode)) { + throw new PlanningException("Unsupported name resolving level: " + mode.name()); } - return resolverMap.get(level).resolve(plan, block, column); + return resolverMap.get(mode).resolve(plan, block, column); } /** diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java new file mode 100644 index 0000000000..5839a556c8 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.nameresolver; + +/** + * + *

Motivation

+ * + * Please take a look at the following example query: + * + * 
+ *   select (l_orderkey + l_orderkey) l_orderkey from lineitem where l_orderkey > 2 order by l_orderkey;
+ *
+ * + * Although l_orderkey seems to be ambiguous, the above usages are available in commercial DBMSs. + * In order to eliminate the ambiguity, Tajo follows the behaviors of PostgreSQL. + * + *

Resolving Modes

+ * + * From the behaviors of PostgreSQL, we found that there are three kinds of name resolving modes. + * Each definition is as follows: + * + *
    + *
  • RELS_ONLY finds a column from the relations in the current block. + *
  • RELS_AND_SUBEXPRS finds a column from the all relations in the current block and + * from aliased temporal fields; a temporal field means an explicitly aliased expression. If there are duplicated + * columns in the relation and temporal fields, this level firstly chooses the field in a relation.
    • + *
  • SUBEXPRS_AND_RELS is very similar to RELS_AND_SUBEXPRS. The main difference is that it + * firstly chooses an aliased temporal field instead of the fields in a relation.
    • + *
+ * + *

The relationship between resolving modes and operators

+ * + *
    + *
  • fields in select list are resolved in the REL_ONLY mode.
    • + *
  • fields in WHERE clause are resolved in the RELS_AND_SUBEXPRS mode.
    • + *
  • fields in GROUP BY, HAVING, ORDER BY, and LIMIT are resolved in the SUBEXPRS_AND_RELS mode.
    • + *
+ * + *

Example

+ * + * Please revisit the aforementioned example: + * + * 
+ *   select (l_orderkey + l_orderkey) l_orderkey from lineitem where l_orderkey > 2 order by l_orderkey;
+ *
+ * + * With the above rules and the relationship between modes and operators, we can easily identify which reference + * points to which field. + *
    + *
  1. l_orderkey included in (l_orderkey + l_orderkey) points to the field + * in the relation lineitem.
    2. + *
  2. l_orderkey included in WHERE clause also points to the field in the relation + * lineitem.
    3. + *
  3. l_orderkey included in ORDER BY clause points to the temporal field + * (l_orderkey + l_orderkey).
    4. + *
+ */ +public enum NameResolvingMode { + RELS_ONLY, // finding from only relations + RELS_AND_SUBEXPRS, // finding from relations and subexprs in a place + SUBEXPRS_AND_RELS, // finding from subexprs and relations in a place + LEGACY // Finding in a legacy manner (globally) +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java index 10d0f9390e..396bc1b184 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java @@ -85,7 +85,7 @@ private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.Q List candidates = TUtil.newList(); if (block.getNamedExprsManager().isAliased(qualifiedName)) { String alias = block.getNamedExprsManager().getAlias(canonicalName); - found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolveLevel.LEGACY); + found = resolve(plan, block, new ColumnReferenceExpr(alias), NameResolvingMode.LEGACY); if (found != null) { candidates.add(found); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index e66c67c0bf..7bb619d815 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -39,7 +39,7 @@ import org.apache.tajo.engine.planner.Target; import org.apache.tajo.engine.planner.logical.GroupbyNode; import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.nameresolver.NameResolveLevel; +import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.exception.InternalException; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; @@ -156,7 +156,7 @@ public static EvalNode getRootSelection(String query) throws PlanningException { Selection selection = plan.getRootBlock().getSingletonExpr(OpType.Filter); return planner.getExprAnnotator().createEvalNode(plan, plan.getRootBlock(), selection.getQual(), - NameResolveLevel.RELS_AND_SUBEXPRS); + NameResolvingMode.RELS_AND_SUBEXPRS); } @Test From 78d2865583eea62d91d2a7578a7f577146c82743 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 17 Jul 2014 15:42:48 +0900 Subject: [PATCH 09/10] Changed resolving level to RELS_ONLY in some operator. --- .../org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java | 2 +- .../java/org/apache/tajo/engine/planner/LogicalPlanner.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index cec1738d6b..7cec1f4341 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -472,7 +472,7 @@ public Expr visitColumnReference(PreprocessContext ctx, Stack stack, Colum throws PlanningException { String normalized = NameResolver.resolve(ctx.plan, ctx.currentBlock, expr, - NameResolveLevel.RELS_AND_SUBEXPRS).getQualifiedName(); + NameResolveLevel.RELS_ONLY).getQualifiedName(); expr.setName(normalized); return expr; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index d0a62cd624..33c777e254 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -263,7 +263,7 @@ private void setRawTargets(PlanContext context, Target[] targets, String[] refer for (int i = 0; i < projection.getNamedExprs().length; i++) { NamedExpr namedExpr = projection.getNamedExprs()[i]; EvalNode evalNode = exprAnnotator.createEvalNode(plan, block, namedExpr.getExpr(), - NameResolveLevel.SUBEXPRS_AND_RELS); + NameResolveLevel.RELS_AND_SUBEXPRS); rawTargets[i] = new Target(evalNode, referenceNames[i]); } // it's for debugging or unit testing From cb4f2eb87f30c25f2369fb6b07dff0d8b46bcabf Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 21 Jul 2014 00:57:18 +0900 Subject: [PATCH 10/10] Fixed the explanation about LIMIT clause in NameResolvingMode.java. --- .../tajo/engine/planner/nameresolver/NameResolvingMode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java index 5839a556c8..dcbb5f1b96 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java @@ -48,9 +48,9 @@ *

The relationship between resolving modes and operators

* *
    - *
  • fields in select list are resolved in the REL_ONLY mode.
    • + *
  • fields in select list and LIMIT are resolved in the REL_ONLY mode.
    • *
  • fields in WHERE clause are resolved in the RELS_AND_SUBEXPRS mode.
    • - *
  • fields in GROUP BY, HAVING, ORDER BY, and LIMIT are resolved in the SUBEXPRS_AND_RELS mode.
    • + *
  • fields in GROUP BY, HAVING, and ORDER BY are resolved in the SUBEXPRS_AND_RELS mode.
    • *
* *

Example