From 31d524c51cc29da1c90b2431279ca63f24a19172 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Fri, 11 Jul 2014 11:47:19 +0900 Subject: [PATCH 1/8] TAJO-836: create index support. (jihoon) --- .../apache/tajo/algebra/CreateDatabase.java | 2 +- .../org/apache/tajo/algebra/CreateIndex.java | 129 +++++++++++++++++ .../org/apache/tajo/algebra/CreateTable.java | 2 +- .../java/org/apache/tajo/algebra/Insert.java | 2 +- .../java/org/apache/tajo/algebra/OpType.java | 1 + .../java/org/apache/tajo/conf/TajoConf.java | 2 + .../apache/tajo/engine/parser/SQLParser.g4 | 4 +- .../tajo/engine/parser/SQLAnalyzer.java | 38 +++++ .../tajo/engine/planner/AlgebraVisitor.java | 1 + .../engine/planner/BaseAlgebraVisitor.java | 9 ++ .../planner/BasicLogicalPlanVisitor.java | 14 ++ .../planner/ExplainLogicalPlanVisitor.java | 6 + .../planner/LogicalPlanPreprocessor.java | 12 ++ .../engine/planner/LogicalPlanVisitor.java | 4 + .../tajo/engine/planner/LogicalPlanner.java | 58 +++++++- .../engine/planner/PhysicalPlannerImpl.java | 7 + .../engine/planner/global/GlobalPlanner.java | 14 ++ .../planner/logical/CreateIndexNode.java | 133 ++++++++++++++++++ .../tajo/engine/planner/logical/NodeType.java | 1 + .../planner/physical/StoreIndexExec.java | 112 +++++++++++++++ .../rewrite/ProjectionPushDownRule.java | 5 + .../tajo/worker/TaskAttemptContext.java | 10 +- .../org/apache/tajo/QueryTestCaseBase.java | 3 + .../engine/planner/TestLogicalPlanner.java | 26 +++- .../planner/physical/TestPhysicalPlanner.java | 13 +- .../tajo/engine/query/TestCreateIndex.java | 72 ++++++++++ .../TestCreateIndex/testCreateIndex.sql | 1 + .../testCreateIndexOnExpression.sql | 1 + .../testCreateIndexOnMultiAttrs.sql | 1 + .../testCreateIndexWithCondition.sql | 1 + 30 files changed, 668 insertions(+), 16 deletions(-) create mode 100644 tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java create mode 100644 tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java create mode 100644 tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndex.sql create mode 100644 tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnExpression.sql create mode 100644 tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiAttrs.sql create mode 100644 tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexWithCondition.sql diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateDatabase.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateDatabase.java index 1144b6eda1..6e74d3db33 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateDatabase.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateDatabase.java @@ -32,7 +32,7 @@ public class CreateDatabase extends Expr { private String tablespaceName; @Expose @SerializedName("IfNotExists") private boolean ifNotExists; - @Expose @SerializedName("DatabaseProperties") + @Expose @SerializedName("Properties") private Map params; public CreateDatabase(final String databaseName, final String tablespaceName, final boolean ifNotExists) { diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java new file mode 100644 index 0000000000..5d9ffa9dc5 --- /dev/null +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.algebra; + +import com.google.common.base.Objects; +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; +import org.apache.tajo.algebra.Sort.SortSpec; +import org.apache.tajo.util.TUtil; + +import java.util.Map; + +public class CreateIndex extends UnaryOperator { + @Expose @SerializedName("IsUnique") + private boolean unique = false; + @Expose @SerializedName("IndexName") + private String indexName; + @Expose @SerializedName("SortSpecs") + private SortSpec[] sortSpecs; + @Expose @SerializedName("Properties") + private Map params; + @Expose @SerializedName("IndexMethodSpec") + private IndexMethodSpec methodSpec; + + public CreateIndex(final String indexName, final SortSpec[] sortSpecs) { + super(OpType.CreateIndex); + this.indexName = indexName; + this.sortSpecs = sortSpecs; + this.methodSpec = new IndexMethodSpec("TWO_LEVEL_BIN_TREE"); + } + + public void setUnique(boolean unique) { + this.unique = unique; + } + + public boolean isUnique() { + return this.unique; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getIndexName() { + return indexName; + } + + public void setSortSpecs(SortSpec[] sortSpecs) { + this.sortSpecs = sortSpecs; + } + + public SortSpec[] getSortSpecs() { + return sortSpecs; + } + + public void setParams(Map params) { + this.params = params; + } + + public Map getParams() { + return this.params; + } + + public void setMethodSpec(IndexMethodSpec methodSpec) { + this.methodSpec = methodSpec; + } + + public IndexMethodSpec getMethodSpec() { + return this.methodSpec; + } + + @Override + public int hashCode() { + return Objects.hashCode(unique, indexName, sortSpecs, params, methodSpec); + } + + @Override + boolean equalsTo(Expr expr) { + CreateIndex other = (CreateIndex) expr; + return this.unique == other.unique && + this.indexName.equals(other.indexName) && + TUtil.checkEquals(this.sortSpecs, other.sortSpecs) && + TUtil.checkEquals(this.params, other.params) && + this.methodSpec.equals(other.methodSpec); + } + + public static class IndexMethodSpec { + @Expose @SerializedName("IndexMethodName") + private String name; + + public IndexMethodSpec(final String name) { + this.name = name; + } + + public String getName() { + return this.name; + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o instanceof IndexMethodSpec) { + IndexMethodSpec other = (IndexMethodSpec) o; + return this.name.equals(other.name); + } + return false; + } + } +} diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java index bd04a910d8..31217114eb 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java @@ -43,7 +43,7 @@ public class CreateTable extends Expr { private String location; @Expose @SerializedName("SubPlan") private Expr subquery; - @Expose @SerializedName("TableProperties") + @Expose @SerializedName("Properties") private Map params; @Expose @SerializedName("PartitionMethodDesc") private PartitionMethodDescExpr partition; diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java index ce9b7034a6..35d92cd91e 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java @@ -39,7 +39,7 @@ public class Insert extends Expr { private String location; @Expose @SerializedName("SubPlan") private Expr subquery; - @Expose @SerializedName("InsertParams") + @Expose @SerializedName("Properties") private Map params; public Insert() { diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java index 19c4ab5b2b..2ef22f7246 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java @@ -50,6 +50,7 @@ public enum OpType { DropTable(DropTable.class), AlterTablespace(AlterTablespace.class), AlterTable(AlterTable.class), + CreateIndex(CreateIndex.class), TruncateTable(TruncateTable.class), // Insert or Update diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 6298d27687..1bbb22bbac 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -150,6 +150,8 @@ public static enum ConfVars { SYSTEM_CONF_PATH("tajo.system-conf.path", EMPTY_VALUE), SYSTEM_CONF_REPLICA_COUNT("tajo.system-conf.replica-count", 20), +// INDEX_DIR("tajo.index.rootdir", "file:///tmp/tajo-${user.name}/index"), + // Tajo Master Service Addresses TAJO_MASTER_UMBILICAL_RPC_ADDRESS("tajo.master.umbilical-rpc.address", "localhost:26001"), TAJO_MASTER_CLIENT_RPC_ADDRESS("tajo.master.client-rpc.address", "localhost:26002"), diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index beba248be9..706c9738fb 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -68,8 +68,8 @@ schema_statement ; index_statement - : CREATE (u=UNIQUE)? INDEX n=identifier ON t=table_name (m=method_specifier)? - LEFT_PAREN s=sort_specifier_list RIGHT_PAREN p=param_clause? + : CREATE (u=UNIQUE)? INDEX identifier ON table_name (method_specifier)? + LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)? ; database_definition diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 580ec61e3c..faf0212a38 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -26,7 +26,9 @@ import org.antlr.v4.runtime.tree.TerminalNode; import org.apache.tajo.algebra.*; import org.apache.tajo.algebra.Aggregation.GroupType; +import org.apache.tajo.algebra.CreateIndex.IndexMethodSpec; import org.apache.tajo.algebra.LiteralValue.LiteralType; +import org.apache.tajo.algebra.Sort.SortSpec; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.engine.parser.SQLParser.*; import org.apache.tajo.storage.StorageConstants; @@ -1139,6 +1141,42 @@ public Expr visitTrim_function(SQLParser.Trim_functionContext ctx) { return new FunctionExpr(functionName, params); } + @Override + public Expr visitIndex_statement(SQLParser.Index_statementContext ctx) { + String indexName = ctx.identifier().getText(); + String tableName = ctx.table_name().getText(); + Relation relation = new Relation(tableName); + SortSpec[] sortSpecs = buildSortSpecs(ctx.sort_specifier_list()); + NamedExpr[] targets = new NamedExpr[sortSpecs.length]; + Projection projection = new Projection(); + int i = 0; + for (SortSpec sortSpec : sortSpecs) { + targets[i++] = new NamedExpr(sortSpec.getKey()); + } + projection.setNamedExprs(targets); + projection.setChild(relation); + + CreateIndex createIndex = new CreateIndex(indexName, sortSpecs); + if (checkIfExist(ctx.UNIQUE())) { + createIndex.setUnique(true); + } + if (checkIfExist(ctx.method_specifier())) { + String methodName = ctx.method_specifier().identifier().getText(); + createIndex.setMethodSpec(new IndexMethodSpec(methodName)); + } + if (checkIfExist(ctx.param_clause())) { + Map params = getParams(ctx.param_clause()); + createIndex.setParams(params); + } + if (checkIfExist(ctx.where_clause())) { + Selection selection = visitWhere_clause(ctx.where_clause()); + selection.setChild(relation); + projection.setChild(selection); + } + createIndex.setChild(projection); + return createIndex; + } + @Override public Expr visitDatabase_definition(@NotNull SQLParser.Database_definitionContext ctx) { return new CreateDatabase(ctx.identifier().getText(), null, checkIfExist(ctx.if_not_exists())); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java index 8c3e6067a0..eb8a21f106 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java @@ -48,6 +48,7 @@ public interface AlgebraVisitor { RESULT visitDropTable(CONTEXT ctx, Stack stack, DropTable expr) throws PlanningException; RESULT visitAlterTablespace(CONTEXT ctx, Stack stack, AlterTablespace expr) throws PlanningException; RESULT visitAlterTable(CONTEXT ctx, Stack stack, AlterTable expr) throws PlanningException; + RESULT visitCreateIndex(CONTEXT ctx, Stack stack, CreateIndex expr) throws PlanningException; RESULT visitTruncateTable(CONTEXT ctx, Stack stack, TruncateTable expr) throws PlanningException; // Insert or Update diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java index 24ff2e47bb..b93541b714 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java @@ -118,6 +118,9 @@ public RESULT visit(CONTEXT ctx, Stack stack, Expr expr) throws PlanningEx case AlterTable: current = visitAlterTable(ctx, stack, (AlterTable) expr); break; + case CreateIndex: + current = visitCreateIndex(ctx, stack, (CreateIndex) expr); + break; case TruncateTable: current = visitTruncateTable(ctx, stack, (TruncateTable)expr); break; @@ -470,6 +473,12 @@ public RESULT visitAlterTable(CONTEXT ctx, Stack stack, AlterTable expr) t return null; } + @Override + public RESULT visitCreateIndex(CONTEXT ctx, Stack stack, CreateIndex expr) throws PlanningException { + return null; + } + + @Override public RESULT visitTruncateTable(CONTEXT ctx, Stack stack, TruncateTable expr) throws PlanningException { return null; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java index a4e90b432b..59422f1aeb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner; +import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; import org.apache.tajo.engine.planner.logical.*; import java.util.Stack; @@ -128,6 +129,9 @@ public RESULT visit(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl case ALTER_TABLE: current = visitAlterTable(context, plan, block, (AlterTableNode) node, stack); break; + case CREATE_INDEX: + current = visitCreateIndex(context, plan, block, (CreateIndexNode) node, stack); + break; case TRUNCATE_TABLE: current = visitTruncateTable(context, plan, block, (TruncateTableNode) node, stack); break; @@ -343,6 +347,16 @@ public RESULT visitAlterTable(CONTEXT context, LogicalPlan plan, LogicalPlan.Que return null; } + @Override + public RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, CreateIndexNode node, + Stack stack) throws PlanningException { + RESULT result = null; + stack.push(node); + result = visit(context, plan, block, node.getChild(), stack); + stack.pop(); + return result; + } + @Override public RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node, Stack stack) throws PlanningException { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java index a7e5375c69..97536c5302 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java @@ -230,6 +230,12 @@ public LogicalNode visitInsert(Context context, LogicalPlan plan, LogicalPlan.Qu return node; } + @Override + public LogicalNode visitCreateIndex(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, + CreateIndexNode node, Stack stack) throws PlanningException { + return visitUnaryNode(context, plan, block, node, stack); + } + public static String printDepthString(int maxDepth, DepthString planStr) { StringBuilder output = new StringBuilder(); String pad = new String(new char[planStr.getDepth() * 3]).replace('\0', ' '); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index 4f1218f2ba..9438d3452d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -431,6 +431,18 @@ public LogicalNode visitAlterTable(PreprocessContext ctx, Stack stack, Alt } @Override + public LogicalNode visitCreateIndex(PreprocessContext ctx, Stack stack, CreateIndex expr) + throws PlanningException { + stack.push(expr); + LogicalNode child = visit(ctx, stack, expr.getChild()); + stack.pop(); + + CreateIndexNode createIndex = ctx.plan.createNode(CreateIndexNode.class); + createIndex.setInSchema(child.getOutSchema()); + createIndex.setOutSchema(child.getOutSchema()); + return createIndex; + } + public LogicalNode visitTruncateTable(PreprocessContext ctx, Stack stack, TruncateTable expr) throws PlanningException { TruncateTableNode truncateTableNode = ctx.plan.createNode(TruncateTableNode.class); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java index 0a36610476..57f943ef6c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.planner; +import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; import org.apache.tajo.engine.planner.logical.*; import java.util.Stack; @@ -93,6 +94,9 @@ RESULT visitAlterTablespace(CONTEXT context, LogicalPlan plan, LogicalPlan.Query RESULT visitAlterTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, AlterTableNode node, Stack stack) throws PlanningException; + RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, CreateIndexNode node, + Stack stack) throws PlanningException; + RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node, Stack stack) throws PlanningException; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 80390d3f80..0315cb113c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -34,6 +34,7 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.engine.eval.*; @@ -788,19 +789,23 @@ public SortNode visitSort(PlanContext context, Stack stack, Sort sort) thr // Building sort keys + sortNode.setSortSpecs(annotateSortSpecs(block, referNames, sortSpecs)); + return sortNode; + } + + private static SortSpec[] annotateSortSpecs(QueryBlock block, String [] referNames, Sort.SortSpec[] rawSortSpecs) { + int sortKeyNum = rawSortSpecs.length; Column column; SortSpec [] annotatedSortSpecs = new SortSpec[sortKeyNum]; for (int i = 0; i < sortKeyNum; i++) { if (block.namedExprsMgr.isEvaluated(referNames[i])) { column = block.namedExprsMgr.getTarget(referNames[i]).getNamedColumn(); } else { - throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(sortSpecs)); + throw new IllegalStateException("Unexpected State: " + TUtil.arrayToString(rawSortSpecs)); } - annotatedSortSpecs[i] = new SortSpec(column, sortSpecs[i].isAscending(), sortSpecs[i].isNullFirst()); + annotatedSortSpecs[i] = new SortSpec(column, rawSortSpecs[i].isAscending(), rawSortSpecs[i].isNullFirst()); } - - sortNode.setSortSpecs(annotatedSortSpecs); - return sortNode; + return annotatedSortSpecs; } /*=============================================================================================== @@ -1828,6 +1833,49 @@ public LogicalNode visitAlterTable(PlanContext context, Stack stack, Alter return alterTableNode; } + @Override + public LogicalNode visitCreateIndex(PlanContext context, Stack stack, CreateIndex createIndex) + throws PlanningException { + stack.push(createIndex); + LogicalNode child = visit(context, stack, createIndex.getChild()); + stack.pop(); + + QueryBlock block = context.queryBlock; + CreateIndexNode createIndexNode = block.getNodeFromExpr(createIndex); + if (CatalogUtil.isFQTableName(createIndex.getIndexName())) { + createIndexNode.setIndexName(createIndex.getIndexName()); + } else { + createIndexNode.setIndexName( + CatalogUtil.buildFQName(context.session.getCurrentDatabase(), createIndex.getIndexName())); + } + createIndexNode.setUnique(createIndex.isUnique()); + Sort.SortSpec[] sortSpecs = createIndex.getSortSpecs(); + int sortKeyNum = sortSpecs.length; + String[] referNames = new String[sortKeyNum]; + + ExprNormalizedResult[] normalizedExprList = new ExprNormalizedResult[sortKeyNum]; + for (int i = 0; i < sortKeyNum; i++) { + normalizedExprList[i] = normalizer.normalize(context, sortSpecs[i].getKey()); + } + for (int i = 0; i < sortKeyNum; i++) { + referNames[i] = block.namedExprsMgr.addExpr(normalizedExprList[i].baseExpr); + block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].aggExprs); + block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs); + } + + createIndexNode.setSortSpecs(annotateSortSpecs(block, referNames, sortSpecs)); + createIndexNode.setIndexType(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase())); + + if (createIndex.getParams() != null) { + KeyValueSet keyValueSet = new KeyValueSet(); + keyValueSet.putAll(createIndex.getParams()); + createIndexNode.setOptions(keyValueSet); + } + + createIndexNode.setChild(child); + return createIndexNode; + } + @Override public LogicalNode visitTruncateTable(PlanContext context, Stack stack, TruncateTable truncateTable) throws PlanningException { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index cf02ecd1ef..8f2ede385e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -234,6 +234,13 @@ private PhysicalExec createPlanRecursive(TaskAttemptContext ctx, LogicalNode log leftExec = createIndexScanExec(ctx, indexScanNode); return leftExec; + case CREATE_INDEX: + CreateIndexNode createIndexNode = (CreateIndexNode) logicalNode; + stack.push(createIndexNode); + leftExec = createPlanRecursive(ctx, createIndexNode.getChild(), stack); + stack.pop(); + return new StoreIndexExec(ctx, createIndexNode, leftExec); + default: return null; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 2d0dd10bf4..e603e874d1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -1586,6 +1586,20 @@ public LogicalNode visitInsert(GlobalPlanContext context, LogicalPlan plan, Logi return node; } + + @Override + public LogicalNode visitCreateIndex(GlobalPlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock queryBlock, + CreateIndexNode node, Stack stack) throws PlanningException { + LogicalNode child = super.visitCreateIndex(context, plan, queryBlock, node, stack); + + // Don't separate execution block. CreateIndex is pushed to the first execution block. + ExecutionBlock childBlock = context.execBlockMap.remove(child.getPID()); + node.setChild(childBlock.getPlan()); + childBlock.setPlan(node); + context.execBlockMap.put(node.getPID(), childBlock); + + return node; + } } @SuppressWarnings("unused") diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java new file mode 100644 index 0000000000..1dca608ce0 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.logical; + +import com.google.common.base.Objects; +import com.google.gson.annotations.Expose; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.TUtil; + +import static org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; + +public class CreateIndexNode extends UnaryNode implements Cloneable { + @Expose private boolean isUnique; + @Expose private String indexName; + @Expose private SortSpec[] sortSpecs; + @Expose private IndexMethod indexType = IndexMethod.TWO_LEVEL_BIN_TREE; + @Expose private KeyValueSet options; + + public CreateIndexNode(int pid) { + super(pid, NodeType.CREATE_INDEX); + } + + public void setUnique(boolean unique) { + this.isUnique = unique; + } + + public boolean isUnique() { + return isUnique; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getIndexName() { + return this.indexName; + } + + public void setSortSpecs(SortSpec[] sortSpecs) { + this.sortSpecs = sortSpecs; + } + + public SortSpec[] getSortSpecs() { + return this.sortSpecs; + } + + public void setIndexType(IndexMethod indexType) { + this.indexType = indexType; + } + + public IndexMethod getIndexType() { + return this.indexType; + } + + public void setOptions(KeyValueSet options) { + this.options = options; + } + + public KeyValueSet getOptions() { + return this.options; + } + + @Override + public int hashCode() { + return Objects.hashCode(isUnique, indexName, sortSpecs, indexType, options); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof CreateIndexNode) { + CreateIndexNode other = (CreateIndexNode) obj; + return this.isUnique == other.isUnique && + TUtil.checkEquals(this.indexName, other.indexName) && + TUtil.checkEquals(this.sortSpecs, other.sortSpecs) && + this.indexType.equals(other.indexType) && + TUtil.checkEquals(this.options, other.options); + } + return false; + } + + @Override + public Object clone() throws CloneNotSupportedException { + CreateIndexNode createIndexNode = (CreateIndexNode) super.clone(); + createIndexNode.isUnique = isUnique; + createIndexNode.indexName = indexName; + createIndexNode.sortSpecs = sortSpecs.clone(); + createIndexNode.indexType = indexType; + createIndexNode.options = (KeyValueSet) (options != null ? options.clone() : null); + return createIndexNode; + } + + private String getSortSpecString() { + StringBuilder sb = new StringBuilder("Column [key= "); + for (int i = 0; i < sortSpecs.length; i++) { + sb.append(sortSpecs[i].getSortKey().getQualifiedName()).append(" ") + .append(sortSpecs[i].isAscending() ? "asc" : "desc"); + if(i < sortSpecs.length - 1) { + sb.append(","); + } + } + sb.append("]"); + return sb.toString(); + } + + @Override + public String toString() { + return "CreateIndex (index=" + indexName + ", type=" + indexType.name() + + ", isUnique=" + isUnique + ", " + getSortSpecString() + ")"; + } + + @Override + public PlanString getPlanString() { + return new PlanString(this); + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java index fa1199bc69..2e9e0dac5c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java @@ -55,6 +55,7 @@ public enum NodeType { DROP_TABLE(DropTableNode.class), ALTER_TABLESPACE (AlterTablespaceNode.class), ALTER_TABLE (AlterTableNode.class), + CREATE_INDEX(CreateIndexNode.class), TRUNCATE_TABLE (TruncateTableNode.class); private final Class baseClass; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java new file mode 100644 index 0000000000..91b5322331 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.physical; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.engine.planner.PlannerUtil; +import org.apache.tajo.engine.planner.logical.CreateIndexNode; +import org.apache.tajo.storage.RowStoreUtil; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.TupleComparator; +import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.index.bst.BSTIndex; +import org.apache.tajo.storage.index.bst.BSTIndex.BSTIndexWriter; +import org.apache.tajo.worker.TaskAttemptContext; + +import java.io.IOException; + +public class StoreIndexExec extends UnaryPhysicalExec { + private static final Log LOG = LogFactory.getLog(StoreIndexExec.class); + private BSTIndexWriter indexWriter; + private final CreateIndexNode logicalPlan; + private int[] indexKeys = null; + private Schema keySchema; + private TupleComparator comparator; + + public StoreIndexExec(final TaskAttemptContext context, final CreateIndexNode logicalPlan, + final PhysicalExec child) { + super(context, logicalPlan.getInSchema(), logicalPlan.getOutSchema(), child); + this.logicalPlan = logicalPlan; + } + + @Override + public void init() throws IOException { + super.init(); + + SortSpec[] sortSpecs = logicalPlan.getSortSpecs(); + indexKeys = new int[sortSpecs.length]; + keySchema = PlannerUtil.sortSpecsToSchema(sortSpecs); + + Column col; + for (int i = 0 ; i < sortSpecs.length; i++) { + col = sortSpecs[i].getSortKey(); + indexKeys[i] = inSchema.getColumnId(col.getQualifiedName()); + } + + TajoConf conf = context.getConf(); + + String[] splits = logicalPlan.getIndexName().split("\\."); + Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), splits[0] + "/" + splits[1] + "/" + + context.getUniqueKeyFromFragments()); + System.out.println("exec: " + indexPath); + // TODO: Create factory using reflection + BSTIndex bst = new BSTIndex(conf); + this.comparator = new TupleComparator(keySchema, sortSpecs); + this.indexWriter = bst.getIndexWriter(indexPath, BSTIndex.TWO_LEVEL_INDEX, keySchema, comparator); + this.indexWriter.setLoadNum(100); + this.indexWriter.open(); + } + + @Override + public Tuple next() throws IOException { + Tuple tuple; + Tuple keyTuple; + Tuple prevKeyTuple = null; + long offset; + + while((tuple = child.next()) != null) { + offset = tuple.getOffset(); + keyTuple = new VTuple(keySchema.size()); + RowStoreUtil.project(tuple, keyTuple, indexKeys); + if (prevKeyTuple == null || !prevKeyTuple.equals(keyTuple)) { + indexWriter.write(keyTuple, offset); + prevKeyTuple = keyTuple; + } + } + return null; + } + + @Override + public void close() throws IOException { + super.close(); + + indexWriter.flush(); + IOUtils.cleanup(LOG, indexWriter); + + indexWriter = null; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java index 2bc210cf4d..d92883eab9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java @@ -505,6 +505,11 @@ public LogicalNode visitProjection(Context context, LogicalPlan plan, LogicalPla createTableNode.setChild(child); createTableNode.setInSchema(child.getOutSchema()); break; + case CREATE_INDEX: + CreateIndexNode createIndexNode = (CreateIndexNode) parentNode; + createIndexNode.setChild(child); + createIndexNode.setInSchema(child.getOutSchema()); + break; default: throw new PlanningException("Unexpected Parent Node: " + parentNode.getType()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index e073652bc9..325aceba90 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -275,7 +275,15 @@ public Collection getInputTables() { } return fragmentMap.get(id).toArray(new FragmentProto[fragmentMap.get(id).size()]); } - + + public long getUniqueKeyFromFragments() { + List totalFragments = new ArrayList(); + for (List eachFragments : fragmentMap.values()) { + totalFragments.addAll(eachFragments); + } + return Objects.hashCode(totalFragments.toArray(new FragmentProto[totalFragments.size()])); + } + public int hashCode() { return Objects.hashCode(queryId); } diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 70c73f9cda..bc64bb858b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -533,6 +533,9 @@ private List executeDDL(String ddlFileName, @Nullable String dataFileNam if (isLocalTable) { createdTableGlobalSet.remove(tableName); } + } else if (expr.getType() == OpType.CreateIndex) { + // TODO: index existence check + client.executeQuery(compiled); } else { assertTrue(ddlFilePath + " is not a Create or Drop Table statement", false); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 6d0f7e01f9..2ca85cc545 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -140,14 +140,14 @@ public static void tearDown() throws Exception { "select name, empid, e.deptname, manager from employee as e, dept as dp", // 1 "select name, empid, e.deptname, manager, score from employee as e, dept, score", // 2 "select p.deptname, sumtest(score) from dept as p, score group by p.deptName having sumtest(score) > 30", // 3 - "select p.deptname, score from dept as p, score order by score asc", // 4 + "select p.deptname, score*200 from dept as p, score order by score*10 asc", // 4 "select name from employee where empId = 100", // 5 "select name, score from employee, score", // 6 "select p.deptName, sumtest(score) from dept as p, score group by p.deptName", // 7 "create table store1 as select p.deptName, sumtest(score) from dept as p, score group by p.deptName", // 8 "select deptName, sumtest(score) from score group by deptName having sumtest(score) > 30", // 9 "select 7 + 8 as res1, 8 * 9 as res2, 10 * 10 as res3", // 10 - "create index idx_employee on employee using bitmap (name null first, empId desc) with ('fillfactor' = 70)", // 11 + "create index idx_employee on employee using bitmap_idx (name null first, empId desc) where empid > 100", // 11 "select name, score from employee, score order by score limit 3", // 12 "select length(name), length(deptname), *, empid+10 from employee where empId > 500", // 13 }; @@ -835,6 +835,28 @@ public final void testExprNode() throws PlanningException { assertEquals("res3", col.getSimpleName()); } + @Test + public final void testCreateIndexNode() throws PlanningException { + Expr expr = sqlAnalyzer.parse(QUERIES[11]); + LogicalPlan rootNode = planner.createPlan(session, expr); + LogicalNode plan = rootNode.getRootBlock().getRoot(); + testJsonSerDerObject(plan); + + LogicalRootNode root = (LogicalRootNode) plan; + assertEquals(NodeType.CREATE_INDEX, root.getChild().getType()); + CreateIndexNode createIndexNode = root.getChild(); + + assertEquals(NodeType.PROJECTION, createIndexNode.getChild().getType()); + ProjectionNode projNode = createIndexNode.getChild(); + + assertEquals(NodeType.SELECTION, projNode.getChild().getType()); + SelectionNode selNode = projNode.getChild(); + + assertEquals(NodeType.SCAN, selNode.getChild().getType()); + ScanNode scanNode = selNode.getChild(); + assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); + } + @Test public final void testAsterisk() throws CloneNotSupportedException, PlanningException { Expr expr = sqlAnalyzer.parse(QUERIES[13]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index a823d2bb9b..920c9329b5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -34,6 +34,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; @@ -789,17 +790,23 @@ public final void testEvalExpr() throws IOException, PlanningException { } public final String [] createIndexStmt = { - "create index idx_employee on employee using bst (name null first, empId desc)" + "create index idx_employee on employee using TWO_LEVEL_BIN_TREE (name null first, empId desc)" }; - //@Test + @Test public final void testCreateIndex() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); + Path indexPath = StorageUtil.concatPath(conf.getVar(ConfVars.ROOT_DIR), "default/idx_employee"); + if (sm.getFileSystem().exists(indexPath)) { + sm.getFileSystem().delete(indexPath, true); + } + TaskAttemptContext ctx = new TaskAttemptContext(conf, new QueryContext(), LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); + ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(createIndexStmt[0]); LogicalPlan plan = planner.createPlan(session, context); LogicalNode rootNode = optimizer.optimize(plan); @@ -811,7 +818,7 @@ public final void testCreateIndex() throws IOException, PlanningException { } exec.close(); - FileStatus [] list = sm.getFileSystem().listStatus(StorageUtil.concatPath(workDir, "index")); + FileStatus[] list = sm.getFileSystem().listStatus(indexPath); assertEquals(2, list.length); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java new file mode 100644 index 0000000000..2a66909640 --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.IntegrationTest; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +import static org.junit.Assert.*; + +@Category(IntegrationTest.class) +public class TestCreateIndex extends QueryTestCaseBase { + + public TestCreateIndex() { + super(TajoConstants.DEFAULT_DATABASE_NAME); + } + + private static void assertIndexExist(String indexName) throws IOException { + Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName); + FileSystem fs = indexPath.getFileSystem(conf); + assertTrue(fs.exists(indexPath)); + assertEquals(2, fs.listStatus(indexPath).length); + fs.deleteOnExit(indexPath); + } + + @Test + public final void testCreateIndex() throws Exception { + executeQuery(); + assertIndexExist("l_orderkey_idx"); + } + + @Test + public final void testCreateIndexOnMultiAttrs() throws Exception { + executeQuery(); + assertIndexExist("l_orderkey_partkey_idx"); + } + + @Test + public final void testCreateIndexWithCondition() throws Exception { + executeQuery(); + assertIndexExist("l_orderkey_partkey_lt10_idx"); + } + + @Test + public final void testCreateIndexOnExpression() throws Exception { + executeQuery(); + assertIndexExist("l_orderkey_100_lt10_idx"); + } +} diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndex.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndex.sql new file mode 100644 index 0000000000..1cb8936b30 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndex.sql @@ -0,0 +1 @@ +create index l_orderkey_idx on lineitem (l_orderkey asc null first); \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnExpression.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnExpression.sql new file mode 100644 index 0000000000..0d19cb3f33 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnExpression.sql @@ -0,0 +1 @@ +create index l_orderkey_100_lt10_idx on lineitem (l_orderkey*100 asc null first) where l_orderkey*100 > 10; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiAttrs.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiAttrs.sql new file mode 100644 index 0000000000..3487d2e160 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiAttrs.sql @@ -0,0 +1 @@ +create index l_orderkey_partkey_idx on lineitem (l_orderkey asc, l_partkey desc null last); \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexWithCondition.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexWithCondition.sql new file mode 100644 index 0000000000..ad4ee476c7 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexWithCondition.sql @@ -0,0 +1 @@ +create index l_orderkey_partkey_lt10_idx on lineitem (l_orderkey asc, l_partkey desc) where l_partkey > 10; \ No newline at end of file From d1d4d25fb0d08f0846e758c43caf3e3f3b1bc543 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 2 Oct 2014 09:40:58 +0900 Subject: [PATCH 2/8] tmp --- .../org/apache/tajo/catalog/IndexDesc.java | 21 +++++++------- .../src/main/proto/CatalogProtos.proto | 2 +- .../apache/tajo/catalog/TestIndexDesc.java | 11 ++++---- .../apache/tajo/catalog/CatalogServer.java | 6 ++-- .../tajo/catalog/store/AbstractDBStore.java | 4 +-- .../apache/tajo/catalog/store/MemStore.java | 6 ++-- .../org/apache/tajo/catalog/TestCatalog.java | 8 +++--- .../planner/logical/CreateIndexNode.java | 28 +++++++++++++------ .../planner/physical/StoreIndexExec.java | 7 +++-- 9 files changed, 53 insertions(+), 40 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java index cff20593c2..3f772b7d2a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java @@ -21,6 +21,7 @@ import com.google.common.base.Objects; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; @@ -29,7 +30,7 @@ public class IndexDesc implements ProtoObject, Cloneable { private IndexDescProto.Builder builder; - private String indexName; // required + private Path indexPath; // required private String databaseName; // required private String tableName; // required private Column column; // required @@ -42,10 +43,10 @@ public IndexDesc() { this.builder = IndexDescProto.newBuilder(); } - public IndexDesc(String idxName, String databaseName, String tableName, Column column, + public IndexDesc(Path indexPath, String databaseName, String tableName, Column column, IndexMethod type, boolean isUnique, boolean isClustered, boolean isAscending) { this(); - this.indexName = idxName; + this.indexPath = indexPath; this.databaseName = databaseName; this.tableName = tableName; this.column = column; @@ -56,15 +57,15 @@ public IndexDesc(String idxName, String databaseName, String tableName, Column c } public IndexDesc(IndexDescProto proto) { - this(proto.getIndexName(), + this(new Path(proto.getIndexPath()), proto.getTableIdentifier().getDatabaseName(), proto.getTableIdentifier().getTableName(), new Column(proto.getColumn()), proto.getIndexMethod(), proto.getIsUnique(), proto.getIsClustered(), proto.getIsAscending()); } - public String getIndexName() { - return indexName; + public Path getIndexPath() { + return indexPath; } public String getTableName() { @@ -106,7 +107,7 @@ public IndexDescProto getProto() { } builder.setTableIdentifier(tableIdentifierBuilder.build()); - builder.setIndexName(this.indexName); + builder.setIndexPath(this.indexPath.toString()); builder.setColumn(this.column.getProto()); builder.setIndexMethod(indexMethod); builder.setIsUnique(this.isUnique); @@ -119,7 +120,7 @@ public IndexDescProto getProto() { public boolean equals(Object obj) { if (obj instanceof IndexDesc) { IndexDesc other = (IndexDesc) obj; - return getIndexName().equals(other.getIndexName()) + return getIndexPath().equals(other.getIndexPath()) && getTableName().equals(other.getTableName()) && getColumn().equals(other.getColumn()) && getIndexMethod().equals(other.getIndexMethod()) @@ -132,13 +133,13 @@ && isClustered() == other.isClustered() } public int hashCode() { - return Objects.hashCode(getIndexName(), getTableName(), getColumn(), + return Objects.hashCode(getIndexPath(), getTableName(), getColumn(), getIndexMethod(), isUnique(), isClustered(), isAscending()); } public Object clone() throws CloneNotSupportedException { IndexDesc desc = (IndexDesc) super.clone(); - desc.indexName = indexName; + desc.indexPath = indexPath; desc.tableName = tableName; desc.column = column; desc.indexMethod = indexMethod; diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 367d0b8380..036df8541e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -139,7 +139,7 @@ message FunctionDescProto { message IndexDescProto { required TableIdentifierProto tableIdentifier = 1; - required string indexName = 2; + required string indexPath = 2; required ColumnProto column = 3; required IndexMethod indexMethod = 4; optional bool isUnique = 5 [default = false]; diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java index 3fa8f5399b..34b28433d0 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java @@ -18,6 +18,7 @@ package org.apache.tajo.catalog; +import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.common.TajoDataTypes.Type; @@ -36,15 +37,15 @@ public class TestIndexDesc { static { desc1 = new IndexDesc( - "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), + new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); desc2 = new IndexDesc( - "idx_test2", DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), + new Path("idx_test2"), DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false); desc3 = new IndexDesc( - "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), + new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); } @@ -65,7 +66,7 @@ public void testIndexDescProto() { @Test public void testGetFields() { - assertEquals("idx_test", desc1.getIndexName()); + assertEquals("idx_test", desc1.getIndexPath()); assertEquals("indexed", desc1.getTableName()); assertEquals(new Column("id", Type.INT4), desc1.getColumn()); assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc1.getIndexMethod()); @@ -73,7 +74,7 @@ public void testGetFields() { assertEquals(true, desc1.isClustered()); assertEquals(true, desc1.isAscending()); - assertEquals("idx_test2", desc2.getIndexName()); + assertEquals("idx_test2", desc2.getIndexPath()); assertEquals("indexed", desc2.getTableName()); assertEquals(new Column("score", Type.FLOAT8), desc2.getColumn()); assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc2.getIndexMethod()); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index cf3ea6f627..ccbd68bb8e 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -706,12 +706,12 @@ public BoolProto createIndex(RpcController controller, IndexDescProto indexDesc) try { if (store.existIndexByName( indexDesc.getTableIdentifier().getDatabaseName(), - indexDesc.getIndexName())) { - throw new AlreadyExistsIndexException(indexDesc.getIndexName()); + indexDesc.getIndexPath())) { + throw new AlreadyExistsIndexException(indexDesc.getIndexPath()); } store.createIndex(indexDesc); } catch (Exception e) { - LOG.error("ERROR : cannot add index " + indexDesc.getIndexName(), e); + LOG.error("ERROR : cannot add index " + indexDesc.getIndexPath(), e); LOG.error(indexDesc); throw new ServiceException(e); } finally { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index bcf67741f3..3c80dc7fd5 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1630,7 +1630,7 @@ public void createIndex(final IndexDescProto proto) throws CatalogException { pstmt = conn.prepareStatement(sql); pstmt.setInt(1, databaseId); pstmt.setInt(2, tableId); - pstmt.setString(3, proto.getIndexName()); + pstmt.setString(3, proto.getIndexPath()); pstmt.setString(4, columnName); pstmt.setString(5, proto.getColumn().getDataType().getType().name()); pstmt.setString(6, proto.getIndexMethod().toString()); @@ -1883,7 +1883,7 @@ public IndexDescProto[] getIndexes(String databaseName, final String tableName) private void resultToIndexDescProtoBuilder(IndexDescProto.Builder builder, final ResultSet res) throws SQLException { - builder.setIndexName(res.getString("index_name")); + builder.setIndexPath(res.getString("index_name")); builder.setColumn(indexResultToColumnProto(res)); builder.setIndexMethod(getIndexMethod(res.getString("index_type").trim())); builder.setIsUnique(res.getBoolean("is_unique")); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index ca9916005c..4b9a28258e 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -364,11 +364,11 @@ public void createIndex(IndexDescProto proto) throws CatalogException { Map index = checkAndGetDatabaseNS(indexes, databaseName); Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - if (index.containsKey(proto.getIndexName())) { - throw new AlreadyExistsIndexException(proto.getIndexName()); + if (index.containsKey(proto.getIndexPath())) { + throw new AlreadyExistsIndexException(proto.getIndexPath()); } - index.put(proto.getIndexName(), proto); + index.put(proto.getIndexPath(), proto); indexByColumn.put(proto.getTableIdentifier().getTableName() + "." + CatalogUtil.extractSimpleName(proto.getColumn().getName()), proto); } diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 654736dc70..959c7883fd 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -374,15 +374,15 @@ public void testGetTable() throws Exception { static { desc1 = new IndexDesc( - "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), + new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); desc2 = new IndexDesc( - "idx_test2", DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), + new Path("idx_test2"), DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false); desc3 = new IndexDesc( - "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), + new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); } @@ -438,7 +438,7 @@ public void testAddAndDelIndex() throws Exception { TableDesc desc = prepareTable(); assertTrue(catalog.createTable(desc)); - assertFalse(catalog.existIndexByName("db1", desc1.getIndexName())); + assertFalse(catalog.existIndexByName("db1", desc1.getIndexPath())); assertFalse(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "id")); catalog.createIndex(desc1); assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getIndexName())); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java index 1dca608ce0..83eb73c2db 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java @@ -20,6 +20,7 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.engine.planner.PlanString; import org.apache.tajo.util.KeyValueSet; @@ -29,7 +30,8 @@ public class CreateIndexNode extends UnaryNode implements Cloneable { @Expose private boolean isUnique; - @Expose private String indexName; +// @Expose private String indexName; + @Expose private Path indexPath; @Expose private SortSpec[] sortSpecs; @Expose private IndexMethod indexType = IndexMethod.TWO_LEVEL_BIN_TREE; @Expose private KeyValueSet options; @@ -46,12 +48,20 @@ public boolean isUnique() { return isUnique; } - public void setIndexName(String indexName) { - this.indexName = indexName; +// public void setIndexName(String indexName) { +// this.indexName = indexName; +// } +// +// public String getIndexName() { +// return this.indexName; +// } + + public void setIndexPath(Path indexPath) { + this.indexPath = indexPath; } - public String getIndexName() { - return this.indexName; + public Path getIndexPath() { + return this.indexPath; } public void setSortSpecs(SortSpec[] sortSpecs) { @@ -80,7 +90,7 @@ public KeyValueSet getOptions() { @Override public int hashCode() { - return Objects.hashCode(isUnique, indexName, sortSpecs, indexType, options); + return Objects.hashCode(isUnique, indexPath, sortSpecs, indexType, options); } @Override @@ -88,7 +98,7 @@ public boolean equals(Object obj) { if (obj instanceof CreateIndexNode) { CreateIndexNode other = (CreateIndexNode) obj; return this.isUnique == other.isUnique && - TUtil.checkEquals(this.indexName, other.indexName) && + TUtil.checkEquals(this.indexPath, other.indexPath) && TUtil.checkEquals(this.sortSpecs, other.sortSpecs) && this.indexType.equals(other.indexType) && TUtil.checkEquals(this.options, other.options); @@ -100,7 +110,7 @@ public boolean equals(Object obj) { public Object clone() throws CloneNotSupportedException { CreateIndexNode createIndexNode = (CreateIndexNode) super.clone(); createIndexNode.isUnique = isUnique; - createIndexNode.indexName = indexName; + createIndexNode.indexPath = indexPath; createIndexNode.sortSpecs = sortSpecs.clone(); createIndexNode.indexType = indexType; createIndexNode.options = (KeyValueSet) (options != null ? options.clone() : null); @@ -122,7 +132,7 @@ private String getSortSpecString() { @Override public String toString() { - return "CreateIndex (index=" + indexName + ", type=" + indexType.name() + + return "CreateIndex (indexPath=" + indexPath + ", type=" + indexType.name() + ", isUnique=" + isUnique + ", " + getSortSpecString() + ")"; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java index 91b5322331..7cbea13c53 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java @@ -69,9 +69,10 @@ public void init() throws IOException { TajoConf conf = context.getConf(); - String[] splits = logicalPlan.getIndexName().split("\\."); - Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), splits[0] + "/" + splits[1] + "/" + - context.getUniqueKeyFromFragments()); +// String[] splits = logicalPlan.getIndexName().split("\\."); +// Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), splits[0] + "/" + splits[1] + "/" + +// context.getUniqueKeyFromFragments()); + Path indexPath = logicalPlan.getIndexPath(); System.out.println("exec: " + indexPath); // TODO: Create factory using reflection BSTIndex bst = new BSTIndex(conf); From 7f98fe78613ce6349c94dba559f7e4b4197b18e9 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 2 Oct 2014 17:49:58 +0900 Subject: [PATCH 3/8] test success --- .../org/apache/tajo/catalog/IndexDesc.java | 17 ++++++++++---- .../src/main/proto/CatalogProtos.proto | 13 ++++++----- .../apache/tajo/catalog/TestIndexDesc.java | 12 +++++----- .../tajo/catalog/store/AbstractDBStore.java | 15 ++++++++----- .../main/resources/schemas/derby/indexes.sql | 1 + .../resources/schemas/mariadb/indexes.sql | 1 + .../main/resources/schemas/mysql/indexes.sql | 1 + .../org/apache/tajo/catalog/TestCatalog.java | 22 +++++++++---------- .../tajo/engine/planner/LogicalPlanner.java | 9 ++++++++ .../planner/logical/CreateIndexNode.java | 22 ++++++++++--------- .../planner/physical/StoreIndexExec.java | 2 +- .../planner/physical/TestPhysicalPlanner.java | 2 +- 12 files changed, 73 insertions(+), 44 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java index 3f772b7d2a..e7cd4507e3 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java @@ -29,7 +29,8 @@ public class IndexDesc implements ProtoObject, Cloneable { private IndexDescProto.Builder builder; - + + private String name; private Path indexPath; // required private String databaseName; // required private String tableName; // required @@ -43,9 +44,10 @@ public IndexDesc() { this.builder = IndexDescProto.newBuilder(); } - public IndexDesc(Path indexPath, String databaseName, String tableName, Column column, + public IndexDesc(String name, Path indexPath, String databaseName, String tableName, Column column, IndexMethod type, boolean isUnique, boolean isClustered, boolean isAscending) { this(); + this.name = name; this.indexPath = indexPath; this.databaseName = databaseName; this.tableName = tableName; @@ -57,12 +59,16 @@ public IndexDesc(Path indexPath, String databaseName, String tableName, Column c } public IndexDesc(IndexDescProto proto) { - this(new Path(proto.getIndexPath()), + this(proto.getName(), new Path(proto.getIndexPath()), proto.getTableIdentifier().getDatabaseName(), proto.getTableIdentifier().getTableName(), new Column(proto.getColumn()), proto.getIndexMethod(), proto.getIsUnique(), proto.getIsClustered(), proto.getIsAscending()); } + + public String getName() { + return name; + } public Path getIndexPath() { return indexPath; @@ -107,6 +113,7 @@ public IndexDescProto getProto() { } builder.setTableIdentifier(tableIdentifierBuilder.build()); + builder.setName(this.name); builder.setIndexPath(this.indexPath.toString()); builder.setColumn(this.column.getProto()); builder.setIndexMethod(indexMethod); @@ -121,6 +128,7 @@ public boolean equals(Object obj) { if (obj instanceof IndexDesc) { IndexDesc other = (IndexDesc) obj; return getIndexPath().equals(other.getIndexPath()) + && getName().equals(other.getName()) && getTableName().equals(other.getTableName()) && getColumn().equals(other.getColumn()) && getIndexMethod().equals(other.getIndexMethod()) @@ -133,12 +141,13 @@ && isClustered() == other.isClustered() } public int hashCode() { - return Objects.hashCode(getIndexPath(), getTableName(), getColumn(), + return Objects.hashCode(getName(), getIndexPath(), getTableName(), getColumn(), getIndexMethod(), isUnique(), isClustered(), isAscending()); } public Object clone() throws CloneNotSupportedException { IndexDesc desc = (IndexDesc) super.clone(); + desc.name = name; desc.indexPath = indexPath; desc.tableName = tableName; desc.column = column; diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 036df8541e..38b57a103c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -139,12 +139,13 @@ message FunctionDescProto { message IndexDescProto { required TableIdentifierProto tableIdentifier = 1; - required string indexPath = 2; - required ColumnProto column = 3; - required IndexMethod indexMethod = 4; - optional bool isUnique = 5 [default = false]; - optional bool isClustered = 6 [default = false]; - optional bool isAscending = 7 [default = false]; + required string name = 2; + required string indexPath = 3; + required ColumnProto column = 4; + required IndexMethod indexMethod = 5; + optional bool isUnique = 6 [default = false]; + optional bool isClustered = 7 [default = false]; + optional bool isAscending = 8 [default = false]; } enum IndexMethod { diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java index 34b28433d0..afd88f173d 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java @@ -37,15 +37,15 @@ public class TestIndexDesc { static { desc1 = new IndexDesc( - new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), + "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); desc2 = new IndexDesc( - new Path("idx_test2"), DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), + "idx_test2", new Path("idx_test2"), DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false); desc3 = new IndexDesc( - new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), + "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); } @@ -66,18 +66,20 @@ public void testIndexDescProto() { @Test public void testGetFields() { - assertEquals("idx_test", desc1.getIndexPath()); + assertEquals("idx_test", desc1.getName()); assertEquals("indexed", desc1.getTableName()); assertEquals(new Column("id", Type.INT4), desc1.getColumn()); assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc1.getIndexMethod()); + assertEquals(new Path("idx_test"), desc1.getIndexPath()); assertEquals(true, desc1.isUnique()); assertEquals(true, desc1.isClustered()); assertEquals(true, desc1.isAscending()); - assertEquals("idx_test2", desc2.getIndexPath()); + assertEquals("idx_test2", desc2.getName()); assertEquals("indexed", desc2.getTableName()); assertEquals(new Column("score", Type.FLOAT8), desc2.getColumn()); assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc2.getIndexMethod()); + assertEquals(new Path("idx_test2"), desc2.getIndexPath()); assertEquals(false, desc2.isUnique()); assertEquals(false, desc2.isClustered()); assertEquals(false, desc2.isAscending()); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 3c80dc7fd5..ea1944fbe0 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1618,7 +1618,8 @@ public void createIndex(final IndexDescProto proto) throws CatalogException { String sql = "INSERT INTO " + TB_INDEXES + " (" + COL_DATABASES_PK + ", " + COL_TABLES_PK + ", INDEX_NAME, " + - "COLUMN_NAME, DATA_TYPE, INDEX_TYPE, IS_UNIQUE, IS_CLUSTERED, IS_ASCENDING) VALUES (?,?,?,?,?,?,?,?,?)"; + "COLUMN_NAME, DATA_TYPE, INDEX_TYPE, PATH, IS_UNIQUE, IS_CLUSTERED, IS_ASCENDING) " + + "VALUES (?,?,?,?,?,?,?,?,?,?)"; if (LOG.isDebugEnabled()) { LOG.debug(sql); @@ -1634,9 +1635,10 @@ public void createIndex(final IndexDescProto proto) throws CatalogException { pstmt.setString(4, columnName); pstmt.setString(5, proto.getColumn().getDataType().getType().name()); pstmt.setString(6, proto.getIndexMethod().toString()); - pstmt.setBoolean(7, proto.hasIsUnique() && proto.getIsUnique()); - pstmt.setBoolean(8, proto.hasIsClustered() && proto.getIsClustered()); - pstmt.setBoolean(9, proto.hasIsAscending() && proto.getIsAscending()); + pstmt.setString(7, proto.getIndexPath().toString()); + pstmt.setBoolean(8, proto.hasIsUnique() && proto.getIsUnique()); + pstmt.setBoolean(9, proto.hasIsClustered() && proto.getIsClustered()); + pstmt.setBoolean(10, proto.hasIsAscending() && proto.getIsAscending()); pstmt.executeUpdate(); conn.commit(); } catch (SQLException se) { @@ -1690,7 +1692,7 @@ public static String getTableName(Connection conn, int tableId) throws SQLExcept } final static String GET_INDEXES_SQL = - "SELECT " + COL_TABLES_PK + ", INDEX_NAME, COLUMN_NAME, DATA_TYPE, INDEX_TYPE, IS_UNIQUE, " + + "SELECT " + COL_TABLES_PK + ", INDEX_NAME, COLUMN_NAME, DATA_TYPE, INDEX_TYPE, PATH, IS_UNIQUE, " + "IS_CLUSTERED, IS_ASCENDING FROM " + TB_INDEXES; @@ -1883,9 +1885,10 @@ public IndexDescProto[] getIndexes(String databaseName, final String tableName) private void resultToIndexDescProtoBuilder(IndexDescProto.Builder builder, final ResultSet res) throws SQLException { - builder.setIndexPath(res.getString("index_name")); + builder.setName(res.getString("index_name")); builder.setColumn(indexResultToColumnProto(res)); builder.setIndexMethod(getIndexMethod(res.getString("index_type").trim())); + builder.setIndexPath(res.getString("path")); builder.setIsUnique(res.getBoolean("is_unique")); builder.setIsClustered(res.getBoolean("is_clustered")); builder.setIsAscending(res.getBoolean("is_ascending")); diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql index c4cfc256d1..5d5a041759 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql @@ -5,6 +5,7 @@ CREATE TABLE INDEXES ( COLUMN_NAME VARCHAR(128) NOT NULL, DATA_TYPE VARCHAR(128) NOT NULL, INDEX_TYPE CHAR(32) NOT NULL, + PATH VARCHAR(4096), IS_UNIQUE BOOLEAN NOT NULL, IS_CLUSTERED BOOLEAN NOT NULL, IS_ASCENDING BOOLEAN NOT NULL, diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql index 62feb3630e..33bf0f5366 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql @@ -5,6 +5,7 @@ CREATE TABLE INDEXES ( COLUMN_NAME VARCHAR(128) NOT NULL, DATA_TYPE VARCHAR(128) NOT NULL, INDEX_TYPE CHAR(32) NOT NULL, + PATH VARCHAR(4096), IS_UNIQUE BOOLEAN NOT NULL, IS_CLUSTERED BOOLEAN NOT NULL, IS_ASCENDING BOOLEAN NOT NULL, diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql index 62feb3630e..33bf0f5366 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql @@ -5,6 +5,7 @@ CREATE TABLE INDEXES ( COLUMN_NAME VARCHAR(128) NOT NULL, DATA_TYPE VARCHAR(128) NOT NULL, INDEX_TYPE CHAR(32) NOT NULL, + PATH VARCHAR(4096), IS_UNIQUE BOOLEAN NOT NULL, IS_CLUSTERED BOOLEAN NOT NULL, IS_ASCENDING BOOLEAN NOT NULL, diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 959c7883fd..f5ae5c45f9 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -374,15 +374,15 @@ public void testGetTable() throws Exception { static { desc1 = new IndexDesc( - new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), + "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); desc2 = new IndexDesc( - new Path("idx_test2"), DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), + "idx_test2", new Path("idx_test2"), DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false); desc3 = new IndexDesc( - new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), + "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); } @@ -438,23 +438,23 @@ public void testAddAndDelIndex() throws Exception { TableDesc desc = prepareTable(); assertTrue(catalog.createTable(desc)); - assertFalse(catalog.existIndexByName("db1", desc1.getIndexPath())); + assertFalse(catalog.existIndexByName("db1", desc1.getName())); assertFalse(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "id")); catalog.createIndex(desc1); - assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getIndexName())); + assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName())); assertTrue(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "id")); - assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getIndexName())); + assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName())); assertFalse(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "score")); catalog.createIndex(desc2); - assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getIndexName())); + assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName())); assertTrue(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "score")); - catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getIndexName()); - assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getIndexName())); - catalog.dropIndex(DEFAULT_DATABASE_NAME, desc2.getIndexName()); - assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getIndexName())); + catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getName()); + assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName())); + catalog.dropIndex(DEFAULT_DATABASE_NAME, desc2.getName()); + assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getName())); catalog.dropTable(desc.getName()); assertFalse(catalog.existsTable(desc.getName())); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index e38e47d6bb..5b94326837 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -36,6 +36,8 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.exception.VerifyException; @@ -1885,6 +1887,11 @@ public LogicalNode visitAlterTable(PlanContext context, Stack stack, Alter return alterTableNode; } + private static Path getIndexPath(PlanContext context, String databaseName, String indexName) { + return new Path(TajoConf.getWarehouseDir(context.queryContext.getConf()), + databaseName + "/" + indexName + "/"); + } + @Override public LogicalNode visitCreateIndex(PlanContext context, Stack stack, CreateIndex createIndex) throws PlanningException { @@ -1917,6 +1924,8 @@ public LogicalNode visitCreateIndex(PlanContext context, Stack stack, Crea createIndexNode.setSortSpecs(annotateSortSpecs(block, referNames, sortSpecs)); createIndexNode.setIndexType(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase())); + createIndexNode.setIndexPath(getIndexPath(context, context.queryContext.getCurrentDatabase(), + createIndex.getIndexName())); if (createIndex.getParams() != null) { KeyValueSet keyValueSet = new KeyValueSet(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java index 83eb73c2db..b678a2eeb5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java @@ -30,7 +30,7 @@ public class CreateIndexNode extends UnaryNode implements Cloneable { @Expose private boolean isUnique; -// @Expose private String indexName; + @Expose private String indexName; @Expose private Path indexPath; @Expose private SortSpec[] sortSpecs; @Expose private IndexMethod indexType = IndexMethod.TWO_LEVEL_BIN_TREE; @@ -48,13 +48,13 @@ public boolean isUnique() { return isUnique; } -// public void setIndexName(String indexName) { -// this.indexName = indexName; -// } -// -// public String getIndexName() { -// return this.indexName; -// } + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getIndexName() { + return this.indexName; + } public void setIndexPath(Path indexPath) { this.indexPath = indexPath; @@ -90,7 +90,7 @@ public KeyValueSet getOptions() { @Override public int hashCode() { - return Objects.hashCode(isUnique, indexPath, sortSpecs, indexType, options); + return Objects.hashCode(isUnique, indexName, indexPath, sortSpecs, indexType, options); } @Override @@ -98,6 +98,7 @@ public boolean equals(Object obj) { if (obj instanceof CreateIndexNode) { CreateIndexNode other = (CreateIndexNode) obj; return this.isUnique == other.isUnique && + TUtil.checkEquals(this.indexName, other.indexName) && TUtil.checkEquals(this.indexPath, other.indexPath) && TUtil.checkEquals(this.sortSpecs, other.sortSpecs) && this.indexType.equals(other.indexType) && @@ -110,6 +111,7 @@ public boolean equals(Object obj) { public Object clone() throws CloneNotSupportedException { CreateIndexNode createIndexNode = (CreateIndexNode) super.clone(); createIndexNode.isUnique = isUnique; + createIndexNode.indexName = indexName; createIndexNode.indexPath = indexPath; createIndexNode.sortSpecs = sortSpecs.clone(); createIndexNode.indexType = indexType; @@ -132,7 +134,7 @@ private String getSortSpecString() { @Override public String toString() { - return "CreateIndex (indexPath=" + indexPath + ", type=" + indexType.name() + + return "CreateIndex (indexName=" + indexName + ", indexPath=" + indexPath + ", type=" + indexType.name() + ", isUnique=" + isUnique + ", " + getSortSpecString() + ")"; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java index 7cbea13c53..3ee5125bab 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java @@ -72,7 +72,7 @@ public void init() throws IOException { // String[] splits = logicalPlan.getIndexName().split("\\."); // Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), splits[0] + "/" + splits[1] + "/" + // context.getUniqueKeyFromFragments()); - Path indexPath = logicalPlan.getIndexPath(); + Path indexPath = new Path(logicalPlan.getIndexPath(), ""+context.getUniqueKeyFromFragments()); System.out.println("exec: " + indexPath); // TODO: Create factory using reflection BSTIndex bst = new BSTIndex(conf); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 8e6dce3cd7..d687f50636 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -984,7 +984,7 @@ public final void testCreateIndex() throws IOException, PlanningException { Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); //<<<<<<< HEAD - Path indexPath = StorageUtil.concatPath(conf.getVar(ConfVars.ROOT_DIR), "default/idx_employee"); + Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee"); if (sm.getFileSystem().exists(indexPath)) { sm.getFileSystem().delete(indexPath, true); } From 375eb1ab2d457f577cb22a2239eec017fa7b6409 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Fri, 3 Oct 2014 13:08:48 +0900 Subject: [PATCH 4/8] remove commented out codes --- .../main/java/org/apache/tajo/catalog/CatalogServer.java | 6 +++--- .../java/org/apache/tajo/catalog/store/AbstractDBStore.java | 4 ++-- .../main/java/org/apache/tajo/catalog/store/MemStore.java | 6 +++--- .../src/main/java/org/apache/tajo/conf/TajoConf.java | 2 -- .../apache/tajo/engine/planner/LogicalPlanPreprocessor.java | 4 ---- .../apache/tajo/engine/planner/physical/StoreIndexExec.java | 4 ---- .../tajo/engine/planner/physical/TestPhysicalPlanner.java | 4 ---- 7 files changed, 8 insertions(+), 22 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index ccbd68bb8e..24e29cb5c5 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -706,12 +706,12 @@ public BoolProto createIndex(RpcController controller, IndexDescProto indexDesc) try { if (store.existIndexByName( indexDesc.getTableIdentifier().getDatabaseName(), - indexDesc.getIndexPath())) { - throw new AlreadyExistsIndexException(indexDesc.getIndexPath()); + indexDesc.getName())) { + throw new AlreadyExistsIndexException(indexDesc.getName()); } store.createIndex(indexDesc); } catch (Exception e) { - LOG.error("ERROR : cannot add index " + indexDesc.getIndexPath(), e); + LOG.error("ERROR : cannot add index " + indexDesc.getName(), e); LOG.error(indexDesc); throw new ServiceException(e); } finally { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index ea1944fbe0..0b1be86e51 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1631,11 +1631,11 @@ public void createIndex(final IndexDescProto proto) throws CatalogException { pstmt = conn.prepareStatement(sql); pstmt.setInt(1, databaseId); pstmt.setInt(2, tableId); - pstmt.setString(3, proto.getIndexPath()); + pstmt.setString(3, proto.getName()); pstmt.setString(4, columnName); pstmt.setString(5, proto.getColumn().getDataType().getType().name()); pstmt.setString(6, proto.getIndexMethod().toString()); - pstmt.setString(7, proto.getIndexPath().toString()); + pstmt.setString(7, proto.getIndexPath()); pstmt.setBoolean(8, proto.hasIsUnique() && proto.getIsUnique()); pstmt.setBoolean(9, proto.hasIsClustered() && proto.getIsClustered()); pstmt.setBoolean(10, proto.hasIsAscending() && proto.getIsAscending()); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 4b9a28258e..0e0d544153 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -364,11 +364,11 @@ public void createIndex(IndexDescProto proto) throws CatalogException { Map index = checkAndGetDatabaseNS(indexes, databaseName); Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - if (index.containsKey(proto.getIndexPath())) { - throw new AlreadyExistsIndexException(proto.getIndexPath()); + if (index.containsKey(proto.getName())) { + throw new AlreadyExistsIndexException(proto.getName()); } - index.put(proto.getIndexPath(), proto); + index.put(proto.getName(), proto); indexByColumn.put(proto.getTableIdentifier().getTableName() + "." + CatalogUtil.extractSimpleName(proto.getColumn().getName()), proto); } diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index d4807cfbb5..b5a9b506cb 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -153,8 +153,6 @@ public static enum ConfVars implements ConfigKey { SYSTEM_CONF_PATH("tajo.system-conf.path", EMPTY_VALUE), SYSTEM_CONF_REPLICA_COUNT("tajo.system-conf.replica-count", 20), -// INDEX_DIR("tajo.index.rootdir", "file:///tmp/tajo-${user.name}/index"), - // Tajo Master Service Addresses TAJO_MASTER_UMBILICAL_RPC_ADDRESS("tajo.master.umbilical-rpc.address", "localhost:26001"), TAJO_MASTER_CLIENT_RPC_ADDRESS("tajo.master.client-rpc.address", "localhost:26002"), diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index d6dc1c26d6..154ff95218 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -440,7 +440,6 @@ public LogicalNode visitAlterTable(LogicalPlanner.PlanContext ctx, Stack s } @Override -//<<<<<<< HEAD public LogicalNode visitCreateIndex(LogicalPlanner.PlanContext ctx, Stack stack, CreateIndex expr) throws PlanningException { stack.push(expr); @@ -453,10 +452,7 @@ public LogicalNode visitCreateIndex(LogicalPlanner.PlanContext ctx, Stack return createIndex; } -// public LogicalNode visitTruncateTable(PreprocessContext ctx, Stack stack, TruncateTable expr) -//======= public LogicalNode visitTruncateTable(LogicalPlanner.PlanContext ctx, Stack stack, TruncateTable expr) -//>>>>>>> 644b7cd991ea402115c6dc1d198e7f1d7f41771b throws PlanningException { TruncateTableNode truncateTableNode = ctx.plan.createNode(TruncateTableNode.class); return truncateTableNode; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java index 3ee5125bab..9edc1b9b1a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java @@ -68,10 +68,6 @@ public void init() throws IOException { } TajoConf conf = context.getConf(); - -// String[] splits = logicalPlan.getIndexName().split("\\."); -// Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), splits[0] + "/" + splits[1] + "/" + -// context.getUniqueKeyFromFragments()); Path indexPath = new Path(logicalPlan.getIndexPath(), ""+context.getUniqueKeyFromFragments()); System.out.println("exec: " + indexPath); // TODO: Create factory using reflection diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index d687f50636..4587d1a1b6 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -983,16 +983,12 @@ public final void testCreateIndex() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); -//<<<<<<< HEAD Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee"); if (sm.getFileSystem().exists(indexPath)) { sm.getFileSystem().delete(indexPath, true); } TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), -//======= -// TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), -//>>>>>>> 644b7cd991ea402115c6dc1d198e7f1d7f41771b LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan), new FileFragment[] {frags[0]}, workDir); ctx.setEnforcer(new Enforcer()); From f2f8981ea63027e10df18a3b16b29148f2d7b36a Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Fri, 14 Nov 2014 13:55:46 +0900 Subject: [PATCH 5/8] index support --- .../tajo/engine/planner/physical/StoreIndexExec.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java index be1af5065e..0592217b5d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java @@ -28,10 +28,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.plan.logical.CreateIndexNode; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.RowStoreUtil; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.TupleComparator; -import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.*; import org.apache.tajo.storage.index.bst.BSTIndex; import org.apache.tajo.storage.index.bst.BSTIndex.BSTIndexWriter; import org.apache.tajo.worker.TaskAttemptContext; @@ -71,7 +68,7 @@ public void init() throws IOException { System.out.println("exec: " + indexPath); // TODO: Create factory using reflection BSTIndex bst = new BSTIndex(conf); - this.comparator = new TupleComparator(keySchema, sortSpecs); + this.comparator = new BaseTupleComparator(keySchema, sortSpecs); this.indexWriter = bst.getIndexWriter(indexPath, BSTIndex.TWO_LEVEL_INDEX, keySchema, comparator); this.indexWriter.setLoadNum(100); this.indexWriter.open(); From 071c5d05dff157e47d1bb02e79de88a9577c3346 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Fri, 9 Jan 2015 01:13:52 +0900 Subject: [PATCH 6/8] TAJO-838: Improve query planner to utilize index. (jihoon) --- .../org/apache/tajo/algebra/DropIndex.java | 54 ++++ .../java/org/apache/tajo/algebra/OpType.java | 1 + .../tajo/catalog/AbstractCatalogClient.java | 120 ++++++-- .../src/main/proto/CatalogProtocol.proto | 6 +- .../apache/tajo/catalog/CatalogConstants.java | 1 + .../apache/tajo/catalog/CatalogService.java | 24 +- .../org/apache/tajo/catalog/CatalogUtil.java | 46 +++- .../org/apache/tajo/catalog/IndexDesc.java | 132 ++++----- .../org/apache/tajo/catalog/IndexMeta.java | 180 ++++++++++++ .../org/apache/tajo/catalog/TableDesc.java | 4 - .../exception/NoSuchIndexException.java | 2 +- .../src/main/proto/CatalogProtos.proto | 36 +-- .../apache/tajo/catalog/TestIndexDesc.java | 58 ++-- .../tajo/catalog/store/HCatalogStore.java | 62 +++-- .../apache/tajo/catalog/CatalogServer.java | 93 +++++-- .../dictionary/IndexesTableDescriptor.java | 8 +- .../tajo/catalog/store/AbstractDBStore.java | 186 ++++++++----- .../tajo/catalog/store/CatalogStore.java | 29 +- .../apache/tajo/catalog/store/MemStore.java | 120 ++++---- .../main/resources/schemas/derby/derby.xml | 27 +- .../resources/schemas/mariadb/indexes.sql | 14 +- .../main/resources/schemas/mysql/indexes.sql | 14 +- .../main/resources/schemas/oracle/indexes.sql | 18 +- .../schemas/postgresql/postgresql.xml | 29 +- .../org/apache/tajo/catalog/TestCatalog.java | 76 +++--- .../org/apache/tajo/cli/tsql/TajoCli.java | 12 +- .../cli/tsql/commands/DescTableCommand.java | 19 ++ .../tajo/client/CatalogAdminClient.java | 16 ++ .../tajo/client/CatalogAdminClientImpl.java | 138 +++++++++- .../apache/tajo/client/QueryClientImpl.java | 40 +-- .../org/apache/tajo/client/QueryStatus.java | 8 +- .../apache/tajo/client/SessionConnection.java | 12 +- .../apache/tajo/client/TajoClientImpl.java | 37 +++ tajo-client/src/main/proto/ClientProtos.proto | 71 +++-- .../main/proto/TajoMasterClientProtocol.proto | 9 + .../java/org/apache/tajo/OverridableConf.java | 2 +- .../java/org/apache/tajo/SessionVars.java | 4 + .../java/org/apache/tajo/conf/TajoConf.java | 6 +- .../main/java/org/apache/tajo/util/TUtil.java | 10 + .../apache/tajo/engine/parser/SQLParser.g4 | 9 + .../engine/codegen/ExecutorPreCompiler.java | 7 + .../tajo/engine/parser/SQLAnalyzer.java | 8 +- .../engine/planner/PhysicalPlannerImpl.java | 21 +- .../engine/planner/global/GlobalPlanner.java | 9 + .../planner/physical/BSTIndexScanExec.java | 96 +++++-- .../planner/physical/ProjectionExec.java | 1 + .../planner/physical/StoreIndexExec.java | 13 +- .../utils/test/ErrorInjectionRewriter.java | 5 +- .../org/apache/tajo/master/GlobalEngine.java | 11 +- .../NonForwardQueryResultSystemScanner.java | 46 +--- .../tajo/master/TajoMasterClientService.java | 258 ++++++++++++++---- .../apache/tajo/master/exec/DDLExecutor.java | 109 +++++--- .../tajo/master/exec/QueryExecutor.java | 59 +++- .../apache/tajo/master/querymaster/Query.java | 45 +++ .../master/querymaster/QueryMasterTask.java | 23 +- .../java/org/apache/tajo/util/IPCUtil.java | 44 +++ .../java/org/apache/tajo/util/IndexUtil.java | 149 ---------- .../tajo/webapp/QueryExecutorServlet.java | 8 +- .../tajo/worker/TajoWorkerClientService.java | 27 +- .../java/org/apache/tajo/worker/Task.java | 29 +- .../tajo/worker/TaskAttemptContext.java | 13 +- .../apache/tajo/engine/eval/ExprTestBase.java | 4 +- .../engine/planner/TestLogicalOptimizer.java | 2 +- .../engine/planner/TestLogicalPlanner.java | 8 +- .../planner/global/TestBroadcastJoinPlan.java | 28 +- .../planner/physical/TestBSTIndexExec.java | 206 -------------- .../physical/TestHashAntiJoinExec.java | 6 +- .../physical/TestHashSemiJoinExec.java | 6 +- .../planner/physical/TestPhysicalPlanner.java | 5 +- .../engine/planner/physical/TestSortExec.java | 6 +- .../tajo/engine/query/TestCreateIndex.java | 44 ++- .../tajo/engine/query/TestIndexScan.java | 119 ++++++++ .../engine/query/TestTablePartitions.java | 8 +- .../tajo/master/TestExecutionBlockCursor.java | 2 +- .../apache/tajo/master/TestGlobalPlanner.java | 2 +- .../master/querymaster/TestKillQuery.java | 2 +- .../worker/TestRangeRetrieverHandler.java | 2 +- .../testCreateIndexOnMultiExprs.sql | 1 + .../TestIndexScan/testOnMultipleExprs.result | 3 + .../TestIndexScan/testOnMultipleKeys.result | 3 + .../TestIndexScan/testOnMultipleKeys2.result | 3 + .../testOnSortedNonUniqueKeys.result | 4 + .../testOnUnsortedTextKeys.result | 3 + .../TestIndexScan/testWithGroupBy.result | 3 + .../results/TestIndexScan/testWithJoin.result | 4 + .../results/TestIndexScan/testWithSort.result | 4 + .../TestTajoCli/testHelpSessionVars.result | 2 + tajo-docs/src/main/sphinx/index.rst | 1 + .../src/main/sphinx/index/future_work.rst | 8 + .../src/main/sphinx/index/how_to_use.rst | 69 +++++ tajo-docs/src/main/sphinx/index/types.rst | 7 + tajo-docs/src/main/sphinx/index_overview.rst | 20 ++ .../src/main/sphinx/sql_language/ddl.rst | 33 ++- .../apache/tajo/plan/LogicalOptimizer.java | 14 +- .../org/apache/tajo/plan/LogicalPlan.java | 28 ++ .../tajo/plan/LogicalPlanPreprocessor.java | 8 +- .../org/apache/tajo/plan/LogicalPlanner.java | 27 +- .../apache/tajo/plan/NamedExprsManager.java | 4 +- .../tajo/plan/algebra/AlgebraVisitor.java | 1 + .../tajo/plan/algebra/BaseAlgebraVisitor.java | 8 + .../apache/tajo/plan/expr/AlgebraicUtil.java | 69 +++++ .../apache/tajo/plan/expr/EvalTreeUtil.java | 6 +- .../tajo/plan/logical/CreateIndexNode.java | 89 +++--- .../tajo/plan/logical/DropIndexNode.java | 92 +++++++ .../tajo/plan/logical/IndexScanNode.java | 88 +++--- .../apache/tajo/plan/logical/NodeType.java | 3 +- .../rewrite/BaseLogicalPlanRewriteEngine.java | 10 +- .../BaseLogicalPlanRewriteRuleProvider.java | 4 +- .../rewrite/LogicalPlanRewriteEngine.java | 3 +- .../plan/rewrite/LogicalPlanRewriteRule.java | 5 +- .../LogicalPlanRewriteRuleContext.java | 65 +++++ .../plan/rewrite/rules/AccessPathInfo.java | 52 ++++ .../rewrite/rules/AccessPathRewriter.java | 129 +++++++++ .../rewrite/rules/FilterPushDownRule.java | 98 ++++++- .../plan/rewrite/rules/IndexScanInfo.java | 113 ++++++++ .../rules/LogicalPlanEqualityTester.java | 8 +- .../rules/PartitionedTableRewriter.java | 10 +- .../rewrite/rules/ProjectionPushDownRule.java | 16 +- .../tajo/plan/rewrite/rules/SeqScanInfo.java | 43 +++ .../plan/serder/LogicalNodeDeserializer.java | 71 ++++- .../plan/serder/LogicalNodeSerializer.java | 74 +++++ .../org/apache/tajo/plan/util/IndexUtil.java | 72 +++++ .../apache/tajo/plan/util/PlannerUtil.java | 35 ++- .../plan/visitor/BasicLogicalPlanVisitor.java | 18 ++ .../visitor/ExplainLogicalPlanVisitor.java | 7 + .../tajo/plan/visitor/LogicalPlanVisitor.java | 6 + tajo-plan/src/main/proto/Plan.proto | 85 ++++-- .../hbase/AddSortForInsertRewriter.java | 8 +- 128 files changed, 3414 insertions(+), 1287 deletions(-) create mode 100644 tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java create mode 100644 tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexMeta.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/util/IPCUtil.java delete mode 100644 tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java delete mode 100644 tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java create mode 100644 tajo-core/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java create mode 100644 tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiExprs.sql create mode 100644 tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleExprs.result create mode 100644 tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys.result create mode 100644 tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys2.result create mode 100644 tajo-core/src/test/resources/results/TestIndexScan/testOnSortedNonUniqueKeys.result create mode 100644 tajo-core/src/test/resources/results/TestIndexScan/testOnUnsortedTextKeys.result create mode 100644 tajo-core/src/test/resources/results/TestIndexScan/testWithGroupBy.result create mode 100644 tajo-core/src/test/resources/results/TestIndexScan/testWithJoin.result create mode 100644 tajo-core/src/test/resources/results/TestIndexScan/testWithSort.result create mode 100644 tajo-docs/src/main/sphinx/index/future_work.rst create mode 100644 tajo-docs/src/main/sphinx/index/how_to_use.rst create mode 100644 tajo-docs/src/main/sphinx/index/types.rst create mode 100644 tajo-docs/src/main/sphinx/index_overview.rst create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropIndexNode.java create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathInfo.java create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/IndexScanInfo.java create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/SeqScanInfo.java create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java new file mode 100644 index 0000000000..5a75e78872 --- /dev/null +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.algebra; + +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; + +public class DropIndex extends Expr { + @Expose @SerializedName("IndexName") + private String indexName; + + public DropIndex(final String indexName) { + super(OpType.DropIndex); + this.indexName = indexName; + } + + @Override + public int hashCode() { + return indexName.hashCode(); + } + + @Override + boolean equalsTo(Expr expr) { + DropIndex other = (DropIndex) expr; + return this.indexName.equals(other.indexName); + } + + @Override + public Object clone() throws CloneNotSupportedException { + DropIndex clone = (DropIndex) super.clone(); + clone.indexName = indexName; + return clone; + } + + public String getIndexName() { + return indexName; + } +} diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java index 32c51dbd82..47fea6419e 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java @@ -54,6 +54,7 @@ public enum OpType { AlterTablespace(AlterTablespace.class), AlterTable(AlterTable.class), CreateIndex(CreateIndex.class), + DropIndex(DropIndex.class), TruncateTable(TruncateTable.class), // Insert or Update diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 8ef1c9aefe..dc12614ac2 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -43,6 +43,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto; import org.apache.tajo.ha.HAServiceUtil; import org.apache.tajo.util.ProtoUtil; +import org.apache.tajo.util.TUtil; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -378,6 +379,24 @@ public List call(NettyClientBase client) throws Exception { } } + @Override + public List getAllIndexes() { + try { + return new ServerCallable>(pool, getCatalogServerAddr(), CatalogProtocol.class, false) { + + @Override + public List call(NettyClientBase client) throws Exception { + CatalogProtocolService.BlockingInterface stub = getStub(client); + GetIndexesProto response = stub.getAllIndexes(null, ProtoUtil.NULL_PROTO); + return response.getIndexList(); + } + }.withRetries(); + } catch (ServiceException e) { + LOG.error(e.getMessage(), e); + return null; + } + } + @Override public final PartitionMethodDesc getPartitionMethod(final String databaseName, final String tableName) { try { @@ -582,17 +601,40 @@ public Boolean call(NettyClientBase client) throws ServiceException { } @Override - public boolean existIndexByColumn(final String databaseName, final String tableName, final String columnName) { + public boolean existIndexByColumns(final String databaseName, final String tableName, final Column [] columns) { + return existIndexByColumnNames(databaseName, tableName, extractColumnNames(columns)); + } + + @Override + public boolean existIndexByColumnNames(final String databaseName, final String tableName, final String [] columnNames) { try { return new ServerCallable(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) { public Boolean call(NettyClientBase client) throws ServiceException { - GetIndexByColumnRequest.Builder builder = GetIndexByColumnRequest.newBuilder(); + GetIndexByColumnNamesRequest.Builder builder = GetIndexByColumnNamesRequest.newBuilder(); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); - builder.setColumnName(columnName); + for (String colunName : columnNames) { + builder.addColumnNames(colunName); + } CatalogProtocolService.BlockingInterface stub = getStub(client); - return stub.existIndexByColumn(null, builder.build()).getValue(); + return stub.existIndexByColumnNames(null, builder.build()).getValue(); + } + }.withRetries(); + } catch (ServiceException e) { + LOG.error(e.getMessage(), e); + return false; + } + } + + @Override + public boolean existIndexesByTable(final String databaseName, final String tableName) { + try { + return new ServerCallable(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) { + public Boolean call(NettyClientBase client) throws ServiceException { + + CatalogProtocolService.BlockingInterface stub = getStub(client); + return stub.existIndexesByTable(null, CatalogUtil.buildTableIdentifier(databaseName, tableName)).getValue(); } }.withRetries(); } catch (ServiceException e) { @@ -621,20 +663,60 @@ public IndexDesc call(NettyClientBase client) throws ServiceException { } } + private static String[] extractColumnNames(Column[] columns) { + String[] columnNames = new String [columns.length]; + for (int i = 0; i < columnNames.length; i++) { + columnNames[i] = columns[i].getSimpleName(); + } + return columnNames; + } + + @Override + public final IndexDesc getIndexByColumns(final String databaseName, + final String tableName, + final Column [] columns) { + return getIndexByColumnNames(databaseName, tableName, extractColumnNames(columns)); + } + @Override - public final IndexDesc getIndexByColumn(final String databaseName, - final String tableName, - final String columnName) { + public final IndexDesc getIndexByColumnNames(final String databaseName, + final String tableName, + final String [] columnNames) { try { return new ServerCallable(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) { public IndexDesc call(NettyClientBase client) throws ServiceException { - GetIndexByColumnRequest.Builder builder = GetIndexByColumnRequest.newBuilder(); + GetIndexByColumnNamesRequest.Builder builder = GetIndexByColumnNamesRequest.newBuilder(); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); - builder.setColumnName(columnName); + for (String columnName : columnNames) { + builder.addColumnNames(columnName); + } CatalogProtocolService.BlockingInterface stub = getStub(client); - return new IndexDesc(stub.getIndexByColumn(null, builder.build())); + return new IndexDesc(stub.getIndexByColumnNames(null, builder.build())); + } + }.withRetries(); + } catch (ServiceException e) { + LOG.error(e.getMessage(), e); + return null; + } + } + + @Override + public final Collection getAllIndexesByTable(final String databaseName, + final String tableName) { + try { + return new ServerCallable>(this.pool, getCatalogServerAddr(), CatalogProtocol.class, false) { + @Override + public Collection call(NettyClientBase client) throws Exception { + TableIdentifierProto proto = CatalogUtil.buildTableIdentifier(databaseName, tableName); + CatalogProtocolService.BlockingInterface stub = getStub(client); + GetAllIndexesResponse response = stub.getAllIndexesByTable(null, proto); + List indexDescs = TUtil.newList(); + for (IndexDescProto descProto : response.getIndexDescList()) { + indexDescs.add(new IndexDesc(descProto)); + } + return indexDescs; } }.withRetries(); } catch (ServiceException e) { @@ -664,24 +746,6 @@ public Boolean call(NettyClientBase client) throws ServiceException { } } - @Override - public List getAllIndexes() { - try { - return new ServerCallable>(pool, getCatalogServerAddr(), CatalogProtocol.class, false) { - - @Override - public List call(NettyClientBase client) throws Exception { - CatalogProtocolService.BlockingInterface stub = getStub(client); - GetIndexesProto response = stub.getAllIndexes(null, ProtoUtil.NULL_PROTO); - return response.getIndexList(); - } - }.withRetries(); - } catch (ServiceException e) { - LOG.error(e.getMessage(), e); - return null; - } - } - @Override public final boolean createFunction(final FunctionDesc funcDesc) { try { diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index cae5d8892e..cbe689fe4d 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -66,9 +66,11 @@ service CatalogProtocolService { rpc createIndex(IndexDescProto) returns (BoolProto); rpc dropIndex(IndexNameProto) returns (BoolProto); rpc existIndexByName(IndexNameProto) returns (BoolProto); - rpc existIndexByColumn(GetIndexByColumnRequest) returns (BoolProto); + rpc existIndexByColumnNames(GetIndexByColumnNamesRequest) returns (BoolProto); + rpc existIndexesByTable(TableIdentifierProto) returns (BoolProto); rpc getIndexByName(IndexNameProto) returns (IndexDescProto); - rpc getIndexByColumn(GetIndexByColumnRequest) returns (IndexDescProto); + rpc getIndexByColumnNames(GetIndexByColumnNamesRequest) returns (IndexDescProto); + rpc getAllIndexesByTable(TableIdentifierProto) returns (GetAllIndexesResponse); rpc getAllIndexes(NullProto) returns (GetIndexesProto); rpc createFunction(FunctionDescProto) returns (BoolProto); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java index 6ec52b9172..c7df801459 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java @@ -52,5 +52,6 @@ public class CatalogConstants { public static final String COL_TABLESPACE_PK = "SPACE_ID"; public static final String COL_DATABASES_PK = "DB_ID"; public static final String COL_TABLES_PK = "TID"; + public static final String COL_INDEXES_PK = "INDEX_ID"; public static final String COL_TABLES_NAME = "TABLE_NAME"; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index 2a5d890ad2..dd26a27efa 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -19,13 +19,7 @@ package org.apache.tajo.catalog; import org.apache.tajo.catalog.partition.PartitionMethodDesc; -import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; -import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto; -import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; +import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.common.TajoDataTypes.DataType; import java.util.Collection; @@ -154,6 +148,8 @@ public interface CatalogService { */ List getAllColumns(); + List getAllIndexes(); + /** * * @return All FunctionDescs @@ -190,15 +186,21 @@ public interface CatalogService { boolean existIndexByName(String databaseName, String indexName); - boolean existIndexByColumn(String databaseName, String tableName, String columnName); + boolean existIndexByColumns(String databaseName, String tableName, Column[] columns); + + boolean existIndexByColumnNames(String databaseName, String tableName, String [] columnNames); + + boolean existIndexesByTable(String databaseName, String tableName); IndexDesc getIndexByName(String databaseName, String indexName); - IndexDesc getIndexByColumn(String databaseName, String tableName, String columnName); + IndexDesc getIndexByColumns(String databaseName, String tableName, Column [] columns); + + IndexDesc getIndexByColumnNames(String databaseName, String tableName, String [] columnNames); + + Collection getAllIndexesByTable(String databaseName, String tableName); boolean dropIndex(String databaseName, String indexName); - - List getAllIndexes(); boolean createFunction(FunctionDesc funcDesc); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index bb15ec1176..a593cd965f 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -26,6 +26,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -39,12 +40,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; -import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import static org.apache.tajo.common.TajoDataTypes.Type; public class CatalogUtil { @@ -868,4 +865,43 @@ public static KeyValueSet newPhysicalProperties(StoreType type) { return options; } + + /** + * Make a unique name by concatenating column names. + * The concatenation is performed in sequence of columns' occurrence in the relation schema. + * + * @param originalSchema original relation schema + * @param columnNames column names which will be unified + * @return unified name + */ + public static String getUnifiedSimpleColumnName(Schema originalSchema, String[] columnNames) { + String[] simpleNames = new String[columnNames.length]; + for (int i = 0; i < simpleNames.length; i++) { + String[] identifiers = columnNames[i].split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP); + simpleNames[i] = identifiers[identifiers.length-1]; + } + Arrays.sort(simpleNames, new ColumnPosComparator(originalSchema)); + StringBuilder sb = new StringBuilder(); + for (String colName : simpleNames) { + sb.append(colName).append("_"); + } + sb.deleteCharAt(sb.length()-1); + return sb.toString(); + } + + /** + * Given column names, compare the position of columns in the relation schema. + */ + public static class ColumnPosComparator implements Comparator { + + private Schema originlSchema; + public ColumnPosComparator(Schema originalSchema) { + this.originlSchema = originalSchema; + } + + @Override + public int compare(String o1, String o2) { + return originlSchema.getColumnId(o1) - originlSchema.getColumnId(o2); + } + } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java index 7dca4eb291..9f64913ad4 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java @@ -21,80 +21,91 @@ import com.google.common.base.Objects; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.common.ProtoObject; -public class IndexDesc implements ProtoObject, Cloneable { - private IndexDescProto.Builder builder; +import java.net.URI; +import java.net.URISyntaxException; - private String name; - private Path indexPath; // required +public class IndexDesc implements ProtoObject, Cloneable { private String databaseName; // required private String tableName; // required - private Column column; // required - private IndexMethod indexMethod; // required - private boolean isUnique = false; // optional [default = false] - private boolean isClustered = false; // optional [default = false] - private boolean isAscending = false; // optional [default = false] - + private IndexMeta indexMeta; // required + public IndexDesc() { } - public IndexDesc(String name, Path indexPath, String databaseName, String tableName, Column column, - IndexMethod type, boolean isUnique, boolean isClustered, boolean isAscending) { + public IndexDesc(String databaseName, String tableName, String indexName, URI indexPath, SortSpec[] keySortSpecs, + IndexMethod type, boolean isUnique, boolean isClustered, Schema targetRelationSchema) { this(); - this.name = name; - this.indexPath = indexPath; - this.databaseName = databaseName; - this.tableName = tableName; - this.column = column; - this.indexMethod = type; - this.isUnique = isUnique; - this.isClustered = isClustered; - this.isAscending = isAscending; + this.set(databaseName, tableName, indexName, indexPath, keySortSpecs, type, isUnique, isClustered, + targetRelationSchema); } public IndexDesc(IndexDescProto proto) { - this(proto.getName(), new Path(proto.getIndexPath()), - proto.getTableIdentifier().getDatabaseName(), - proto.getTableIdentifier().getTableName(), - new Column(proto.getColumn()), - proto.getIndexMethod(), proto.getIsUnique(), proto.getIsClustered(), proto.getIsAscending()); + this(); + + SortSpec[] keySortSpecs = new SortSpec[proto.getKeySortSpecsCount()]; + for (int i = 0; i < keySortSpecs.length; i++) { + keySortSpecs[i] = new SortSpec(proto.getKeySortSpecs(i)); + } + + try { + this.set(proto.getTableIdentifier().getDatabaseName(), + proto.getTableIdentifier().getTableName(), + proto.getIndexName(), new URI(proto.getIndexPath()), + keySortSpecs, + proto.getIndexMethod(), proto.getIsUnique(), proto.getIsClustered(), + new Schema(proto.getTargetRelationSchema())); + } catch (URISyntaxException e) { + e.printStackTrace(); + } } - public String getName() { - return name; + public void set(String databaseName, String tableName, String indexName, URI indexPath, SortSpec[] keySortSpecs, + IndexMethod type, boolean isUnique, boolean isClustered, Schema targetRelationSchema) { + this.databaseName = databaseName; + this.tableName = tableName; + this.indexMeta = new IndexMeta(indexName, indexPath, keySortSpecs, type, isUnique, isClustered, + targetRelationSchema); } - - public Path getIndexPath() { - return indexPath; + + public String getDatabaseName() { + return databaseName; } - + public String getTableName() { return tableName; } + + public String getName() { + return indexMeta.getIndexName(); + } - public Column getColumn() { - return column; + public URI getIndexPath() { + return indexMeta.getIndexPath(); + } + + public SortSpec[] getKeySortSpecs() { + return indexMeta.getKeySortSpecs(); } public IndexMethod getIndexMethod() { - return this.indexMethod; + return indexMeta.getIndexMethod(); } public boolean isClustered() { - return this.isClustered; + return indexMeta.isClustered(); } public boolean isUnique() { - return this.isUnique; + return indexMeta.isUnique(); } - - public boolean isAscending() { - return this.isAscending; + + public Schema getTargetRelationSchema() { + return indexMeta.getTargetRelationSchema(); } @Override @@ -110,13 +121,15 @@ public IndexDescProto getProto() { } builder.setTableIdentifier(tableIdentifierBuilder.build()); - builder.setName(this.name); - builder.setIndexPath(this.indexPath.toString()); - builder.setColumn(this.column.getProto()); - builder.setIndexMethod(indexMethod); - builder.setIsUnique(this.isUnique); - builder.setIsClustered(this.isClustered); - builder.setIsAscending(this.isAscending); + builder.setIndexName(indexMeta.getIndexName()); + builder.setIndexPath(indexMeta.getIndexPath().toString()); + for (SortSpec colSpec : indexMeta.getKeySortSpecs()) { + builder.addKeySortSpecs(colSpec.getProto()); + } + builder.setIndexMethod(indexMeta.getIndexMethod()); + builder.setIsUnique(indexMeta.isUnique()); + builder.setIsClustered(indexMeta.isClustered()); + builder.setTargetRelationSchema(indexMeta.getTargetRelationSchema().getProto()); return builder.build(); } @@ -124,34 +137,23 @@ public IndexDescProto getProto() { public boolean equals(Object obj) { if (obj instanceof IndexDesc) { IndexDesc other = (IndexDesc) obj; - return getIndexPath().equals(other.getIndexPath()) - && getName().equals(other.getName()) + return getDatabaseName().equals(other.getDatabaseName()) && getTableName().equals(other.getTableName()) - && getColumn().equals(other.getColumn()) - && getIndexMethod().equals(other.getIndexMethod()) - && isUnique() == other.isUnique() - && isClustered() == other.isClustered() - && isAscending() == other.isAscending(); + && this.indexMeta.equals(other.indexMeta); } else { return false; } } public int hashCode() { - return Objects.hashCode(getName(), getIndexPath(), getTableName(), getColumn(), - getIndexMethod(), isUnique(), isClustered(), isAscending()); + return Objects.hashCode(databaseName, tableName, indexMeta); } public Object clone() throws CloneNotSupportedException { IndexDesc desc = (IndexDesc) super.clone(); - desc.name = name; - desc.indexPath = indexPath; - desc.tableName = tableName; - desc.column = column; - desc.indexMethod = indexMethod; - desc.isUnique = isUnique; - desc.isClustered = isClustered; - desc.isAscending = isAscending; + desc.databaseName = this.databaseName; + desc.tableName = this.tableName; + desc.indexMeta = (IndexMeta) this.indexMeta.clone(); return desc; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexMeta.java new file mode 100644 index 0000000000..a911055129 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexMeta.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import com.google.common.base.Objects; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.annotations.Expose; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; +import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.TUtil; + +import java.net.URI; +import java.util.Arrays; +import java.util.Comparator; + +/** + * IndexMeta contains meta information of an index. + * Meta information is the name, an index method, a path to the stored location, index keys, and so on. + */ +public class IndexMeta implements Cloneable { + @Expose private String indexName; // index name + @Expose private IndexMethod indexMethod; // index method + @Expose private URI indexPath; // path to the location + @Expose private SortSpec[] keySortSpecs; // index keys. This array should always be sorted + // according to the position in the targetRelationSchema + @Expose private boolean isUnique = false; // unique key or not + @Expose private boolean isClustered = false; // clustered index or not + @Expose private Schema targetRelationSchema; // schema of the indexed relation + @Expose private KeyValueSet options; // index options. TODO: will be added + + public IndexMeta() {} + + public IndexMeta(String indexName, URI indexPath, SortSpec[] keySortSpecs, + IndexMethod type, boolean isUnique, boolean isClustered, + Schema targetRelationSchema) { + this.indexName = indexName; + this.indexPath = indexPath; + this.indexMethod = type; + this.isUnique = isUnique; + this.isClustered = isClustered; + this.targetRelationSchema = targetRelationSchema; + initKeySortSpecs(targetRelationSchema, keySortSpecs); + } + + private void initKeySortSpecs(final Schema targetRelationSchema, final SortSpec[] keySortSpecs) { + this.targetRelationSchema = targetRelationSchema; + this.keySortSpecs = new SortSpec[keySortSpecs.length]; + for (int i = 0; i < keySortSpecs.length; i++) { + this.keySortSpecs[i] = new SortSpec(keySortSpecs[i].getSortKey(), keySortSpecs[i].isAscending(), + keySortSpecs[i].isNullFirst()); + } + Arrays.sort(this.keySortSpecs, new Comparator() { + @Override + public int compare(SortSpec o1, SortSpec o2) { + return targetRelationSchema.getColumnId(o1.getSortKey().getSimpleName()) + - targetRelationSchema.getColumnId(o2.getSortKey().getSimpleName()); + } + }); + } + + public String getIndexName() { + return indexName; + } + + public void setIndexName(final String indexName) { + this.indexName = indexName; + } + + public IndexMethod getIndexMethod() { + return indexMethod; + } + + public void setIndexMethod(final IndexMethod type) { + this.indexMethod = type; + } + + public URI getIndexPath() { + return indexPath; + } + + public void setIndexPath(final URI indexPath) { + this.indexPath = indexPath; + } + + public SortSpec[] getKeySortSpecs() { + return keySortSpecs; + } + + public void setKeySortSpecs(final Schema targetRelationSchema, final SortSpec[] keySortSpecs) { + initKeySortSpecs(targetRelationSchema, keySortSpecs); + } + + public boolean isUnique() { + return isUnique; + } + + public void setUnique(boolean unique) { + this.isUnique = unique; + } + + public boolean isClustered() { + return isClustered; + } + + public void setClustered(boolean clustered) { + this.isClustered = clustered; + } + + public Schema getTargetRelationSchema() { + return targetRelationSchema; + } + + public KeyValueSet getOptions() { + return options; + } + + public void setOptions(KeyValueSet options) { + this.options = options; + } + + @Override + public boolean equals(Object o) { + if (o instanceof IndexMeta) { + IndexMeta other = (IndexMeta) o; + return this.indexName.equals(other.indexName) + && this.indexPath.equals(other.indexPath) + && this.indexMethod.equals(other.indexMethod) + && TUtil.checkEquals(this.keySortSpecs, other.keySortSpecs) + && this.isUnique == other.isUnique + && this.isClustered == other.isClustered + && this.targetRelationSchema.equals(other.targetRelationSchema); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(indexName, indexPath, indexMethod, Objects.hashCode(keySortSpecs), + isUnique, isClustered, targetRelationSchema); + } + + @Override + public Object clone() throws CloneNotSupportedException { + IndexMeta clone = (IndexMeta) super.clone(); + clone.indexName = indexName; + clone.indexPath = indexPath; + clone.indexMethod = indexMethod; + clone.keySortSpecs = new SortSpec[keySortSpecs.length]; + for (int i = 0; i < keySortSpecs.length; i++) { + clone.keySortSpecs[i] = new SortSpec(this.keySortSpecs[i].getProto()); + } + clone.isUnique = this.isUnique; + clone.isClustered = this.isClustered; + clone.targetRelationSchema = this.targetRelationSchema; + return clone; + } + + @Override + public String toString() { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + return gson.toJson(this); + } +} diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java index ec679f9258..f3f71ba237 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java @@ -22,8 +22,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; @@ -37,8 +35,6 @@ import java.net.URI; public class TableDesc implements ProtoObject, GsonObject, Cloneable { - private final Log LOG = LogFactory.getLog(TableDesc.class); - @Expose protected String tableName; // required @Expose protected Schema schema; @Expose protected TableMeta meta; // required diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchIndexException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchIndexException.java index 0bb7e32465..71e6f158b1 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchIndexException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchIndexException.java @@ -24,7 +24,7 @@ public class NoSuchIndexException extends CatalogException { public NoSuchIndexException() { } - public NoSuchIndexException(String databaseName, String columnName) { + public NoSuchIndexException(String databaseName, String [] columnName) { super(String.format("ERROR: index \" %s \" in %s does not exist", columnName, databaseName)); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 6baead9d0f..f10aa42204 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -115,13 +115,13 @@ message NamespaceProto { message IndexDescProto { required TableIdentifierProto tableIdentifier = 1; - required string name = 2; - required string indexPath = 3; - required ColumnProto column = 4; - required IndexMethod indexMethod = 5; - optional bool isUnique = 6 [default = false]; - optional bool isClustered = 7 [default = false]; - optional bool isAscending = 8 [default = false]; + required string indexName = 2; + required IndexMethod indexMethod = 3; + required string indexPath = 4; + repeated SortSpecProto key_sort_specs = 5; + required SchemaProto targetRelationSchema = 6; + optional bool isUnique = 7 [default = false]; + optional bool isClustered = 8 [default = false]; } enum IndexMethod { @@ -152,7 +152,7 @@ message GetColumnsProto { } message GetIndexesProto { - repeated IndexProto index = 1; + repeated IndexDescProto index = 1; } message GetTableOptionsProto { @@ -167,18 +167,6 @@ message GetTablePartitionsProto { repeated TablePartitionProto part = 1; } -message IndexProto { - required int32 dbId = 1; - required int32 tId = 2; - required string indexName = 3; - required string columnName = 4; - required string dataType = 5; - required string indexType = 6; - optional bool isUnique = 7 [default = false]; - optional bool isClustered = 8 [default = false]; - optional bool isAscending = 9 [default = false]; -} - message TableOptionProto { required int32 tid = 1; required KeyValueProto keyval = 2; @@ -192,9 +180,9 @@ message TablePartitionProto { optional string path = 5; } -message GetIndexByColumnRequest { +message GetIndexByColumnNamesRequest { required TableIdentifierProto tableIdentifier = 1; - required string columnName = 2; + repeated string columnNames = 2; } message IndexNameProto { @@ -203,6 +191,10 @@ message IndexNameProto { required string indexName = 3; } +message GetAllIndexesResponse { + repeated IndexDescProto indexDesc = 1; +} + message GetFunctionsResponse { repeated FunctionDescProto functionDesc = 1; } diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java index afd88f173d..247cd41aa9 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java @@ -18,7 +18,6 @@ package org.apache.tajo.catalog; -import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.common.TajoDataTypes.Type; @@ -26,6 +25,9 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.net.URI; +import java.net.URISyntaxException; + import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; @@ -34,23 +36,29 @@ public class TestIndexDesc { static IndexDesc desc1; static IndexDesc desc2; static IndexDesc desc3; - - static { - desc1 = new IndexDesc( - "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), - IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); - - desc2 = new IndexDesc( - "idx_test2", new Path("idx_test2"), DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), - IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false); - - desc3 = new IndexDesc( - "idx_test", new Path("idx_test"), DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), - IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); - } + static Schema relationSchema; @BeforeClass public static void setUp() throws Exception { + relationSchema = new Schema(new Column[]{new Column("id", Type.INT4), + new Column("score", Type.FLOAT8), new Column("name", Type.TEXT)}); + SortSpec[] colSpecs1 = new SortSpec[1]; + colSpecs1[0] = new SortSpec(new Column("id", Type.INT4), true, true); + desc1 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed", + "idx_test", new URI("idx_test"), colSpecs1, + IndexMethod.TWO_LEVEL_BIN_TREE, true, true, relationSchema); + + SortSpec[] colSpecs2 = new SortSpec[1]; + colSpecs2[0] = new SortSpec(new Column("score", Type.FLOAT8), false, false); + desc2 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed", + "idx_test2", new URI("idx_test2"), colSpecs2, + IndexMethod.TWO_LEVEL_BIN_TREE, false, false, relationSchema); + + SortSpec[] colSpecs3 = new SortSpec[1]; + colSpecs3[0] = new SortSpec(new Column("id", Type.INT4), true, true); + desc3 = new IndexDesc(DEFAULT_DATABASE_NAME, "indexed", + "idx_test", new URI("idx_test"), colSpecs3, + IndexMethod.TWO_LEVEL_BIN_TREE, true, true, relationSchema); } @AfterClass @@ -65,24 +73,28 @@ public void testIndexDescProto() { } @Test - public void testGetFields() { + public void testGetFields() throws URISyntaxException { assertEquals("idx_test", desc1.getName()); assertEquals("indexed", desc1.getTableName()); - assertEquals(new Column("id", Type.INT4), desc1.getColumn()); + assertEquals(1, desc1.getKeySortSpecs().length); + assertEquals(new Column("id", Type.INT4), desc1.getKeySortSpecs()[0].getSortKey()); + assertEquals(true, desc1.getKeySortSpecs()[0].isAscending()); + assertEquals(true, desc1.getKeySortSpecs()[0].isNullFirst()); assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc1.getIndexMethod()); - assertEquals(new Path("idx_test"), desc1.getIndexPath()); + assertEquals(new URI("idx_test"), desc1.getIndexPath()); assertEquals(true, desc1.isUnique()); assertEquals(true, desc1.isClustered()); - assertEquals(true, desc1.isAscending()); - + assertEquals("idx_test2", desc2.getName()); assertEquals("indexed", desc2.getTableName()); - assertEquals(new Column("score", Type.FLOAT8), desc2.getColumn()); + assertEquals(1, desc2.getKeySortSpecs().length); + assertEquals(new Column("score", Type.FLOAT8), desc2.getKeySortSpecs()[0].getSortKey()); + assertEquals(false, desc2.getKeySortSpecs()[0].isAscending()); + assertEquals(false, desc2.getKeySortSpecs()[0].isNullFirst()); assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc2.getIndexMethod()); - assertEquals(new Path("idx_test2"), desc2.getIndexPath()); + assertEquals(new URI("idx_test2"), desc2.getIndexPath()); assertEquals(false, desc2.isUnique()); assertEquals(false, desc2.isClustered()); - assertEquals(false, desc2.isAscending()); } @Test diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index 2c3fc6ac1a..78fee1d5e4 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -43,7 +43,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto; -import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; @@ -704,115 +704,135 @@ private void addNewColumn(String databaseName, String tableName, CatalogProtos.C @Override public void addPartitionMethod(CatalogProtos.PartitionMethodProto partitionMethodProto) throws CatalogException { // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public CatalogProtos.PartitionMethodProto getPartitionMethod(String databaseName, String tableName) throws CatalogException { - return null; // TODO - not implemented yet + // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public boolean existPartitionMethod(String databaseName, String tableName) throws CatalogException { - return false; // TODO - not implemented yet + // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public void dropPartitionMethod(String databaseName, String tableName) throws CatalogException { // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public void addPartitions(CatalogProtos.PartitionsProto partitionsProto) throws CatalogException { // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public void addPartition(String databaseName, String tableName, CatalogProtos.PartitionDescProto partitionDescProto) throws CatalogException { - + throw new UnsupportedOperationException(); } @Override public CatalogProtos.PartitionsProto getPartitions(String tableName) throws CatalogException { - return null; // TODO - not implemented yet + // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public CatalogProtos.PartitionDescProto getPartition(String partitionName) throws CatalogException { - return null; // TODO - not implemented yet + // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public void delPartition(String partitionName) throws CatalogException { // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public void dropPartitions(String tableName) throws CatalogException { - + throw new UnsupportedOperationException(); } @Override public final void addFunction(final FunctionDesc func) throws CatalogException { // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public final void deleteFunction(final FunctionDesc func) throws CatalogException { // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public final void existFunction(final FunctionDesc func) throws CatalogException { // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override public final List getAllFunctionNames() throws CatalogException { // TODO - not implemented yet - return null; + throw new UnsupportedOperationException(); } @Override - public void dropIndex(String databaseName, String indexName) throws CatalogException { + public void createIndex(CatalogProtos.IndexDescProto proto) throws CatalogException { // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override - public boolean existIndexByName(String databaseName, String indexName) throws CatalogException { + public void dropIndex(String databaseName, String indexName) throws CatalogException { // TODO - not implemented yet - return false; + throw new UnsupportedOperationException(); } @Override - public CatalogProtos.IndexDescProto[] getIndexes(String databaseName, String tableName) throws CatalogException { + public CatalogProtos.IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException { // TODO - not implemented yet - return null; + throw new UnsupportedOperationException(); } @Override - public void createIndex(CatalogProtos.IndexDescProto proto) throws CatalogException { + public CatalogProtos.IndexDescProto getIndexByColumns(String databaseName, String tableName, String[] columnNames) + throws CatalogException { // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override - public CatalogProtos.IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException { + public boolean existIndexByName(String databaseName, String indexName) throws CatalogException { // TODO - not implemented yet - return null; + throw new UnsupportedOperationException(); } @Override - public CatalogProtos.IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName) + public boolean existIndexByColumns(String databaseName, String tableName, String[] columnNames) throws CatalogException { // TODO - not implemented yet - return null; + throw new UnsupportedOperationException(); + } + + @Override + public List getAllIndexNamesByTable(String databaseName, String tableName) throws CatalogException { + // TODO - not implemented yet + throw new UnsupportedOperationException(); } @Override - public boolean existIndexByColumn(String databaseName, String tableName, String columnName) throws CatalogException { + public boolean existIndexesByTable(String databaseName, String tableName) throws CatalogException { // TODO - not implemented yet - return false; + throw new UnsupportedOperationException(); } @Override @@ -860,7 +880,7 @@ public List getAllDatabases() throws CatalogException { } @Override - public List getAllIndexes() throws CatalogException { + public List getAllIndexes() throws CatalogException { throw new UnsupportedOperationException(); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 3f4d38dc99..a2b8eaf840 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -870,12 +870,12 @@ public BoolProto createIndex(RpcController controller, IndexDescProto indexDesc) try { if (store.existIndexByName( databaseName, - indexDesc.getName())) { - throw new AlreadyExistsIndexException(indexDesc.getName()); + indexDesc.getIndexName())) { + throw new AlreadyExistsIndexException(indexDesc.getIndexName()); } store.createIndex(indexDesc); } catch (Exception e) { - LOG.error("ERROR : cannot add index " + indexDesc.getName(), e); + LOG.error("ERROR : cannot add index " + indexDesc.getIndexName(), e); LOG.error(indexDesc); throw new ServiceException(e); } finally { @@ -903,17 +903,35 @@ public BoolProto existIndexByName(RpcController controller, IndexNameProto reque } @Override - public BoolProto existIndexByColumn(RpcController controller, GetIndexByColumnRequest request) + public BoolProto existIndexByColumnNames(RpcController controller, GetIndexByColumnNamesRequest request) throws ServiceException { TableIdentifierProto identifier = request.getTableIdentifier(); String databaseName = identifier.getDatabaseName(); String tableName = identifier.getTableName(); - String columnName = request.getColumnName(); + List columnNames = request.getColumnNamesList(); rlock.lock(); try { - return store.existIndexByColumn(databaseName, tableName, columnName) ? + return store.existIndexByColumns(databaseName, tableName, + columnNames.toArray(new String[columnNames.size()])) ? + ProtoUtil.TRUE : ProtoUtil.FALSE; + } catch (Exception e) { + LOG.error(e); + return BoolProto.newBuilder().setValue(false).build(); + } finally { + rlock.unlock(); + } + } + + @Override + public BoolProto existIndexesByTable(RpcController controller, TableIdentifierProto request) throws ServiceException { + String databaseName = request.getDatabaseName(); + String tableName = request.getTableName(); + + rlock.lock(); + try { + return store.existIndexesByTable(databaseName, tableName) ? ProtoUtil.TRUE : ProtoUtil.FALSE; } catch (Exception e) { LOG.error(e); @@ -933,7 +951,7 @@ public IndexDescProto getIndexByName(RpcController controller, IndexNameProto re rlock.lock(); try { if (!store.existIndexByName(databaseName, indexName)) { - throw new NoSuchIndexException(databaseName, indexName); + throw new NoSuchIndexException(indexName); } return store.getIndexByName(databaseName, indexName); } catch (Exception e) { @@ -945,22 +963,59 @@ public IndexDescProto getIndexByName(RpcController controller, IndexNameProto re } @Override - public IndexDescProto getIndexByColumn(RpcController controller, GetIndexByColumnRequest request) + public IndexDescProto getIndexByColumnNames(RpcController controller, GetIndexByColumnNamesRequest request) throws ServiceException { TableIdentifierProto identifier = request.getTableIdentifier(); String databaseName = identifier.getDatabaseName(); String tableName = identifier.getTableName(); - String columnName = request.getColumnName(); + List columnNamesList = request.getColumnNamesList(); + String[] columnNames = new String[columnNamesList.size()]; + columnNames = columnNamesList.toArray(columnNames); rlock.lock(); try { - if (!store.existIndexByColumn(databaseName, tableName, columnName)) { - throw new NoSuchIndexException(databaseName, columnName); + if (!store.existIndexByColumns(databaseName, tableName, columnNames)) { + throw new NoSuchIndexException(databaseName, columnNames); + } + return store.getIndexByColumns(databaseName, tableName, columnNames); + } catch (Exception e) { + LOG.error("ERROR: cannot get index for " + tableName + "." + columnNames, e); + return null; + } finally { + rlock.unlock(); + } + } + + @Override + public GetAllIndexesResponse getAllIndexesByTable(RpcController controller, TableIdentifierProto request) + throws ServiceException { + rlock.lock(); + String databaseName = request.getDatabaseName(); + String tableName = request.getTableName(); + try { + GetAllIndexesResponse.Builder builder = GetAllIndexesResponse.newBuilder(); + for (String eachIndexName : store.getAllIndexNamesByTable(databaseName, tableName)) { + builder.addIndexDesc(store.getIndexByName(databaseName, eachIndexName)); } - return store.getIndexByColumn(databaseName, tableName, columnName); + return builder.build(); } catch (Exception e) { - LOG.error("ERROR : cannot get index for " + tableName + "." + columnName, e); + LOG.error("ERROR: cannot get all indexes for " + databaseName + "." + tableName, e); + return null; + } finally { + rlock.unlock(); + } + } + + @Override + public GetIndexesProto getAllIndexes(RpcController controller, NullProto request) throws ServiceException { + rlock.lock(); + try { + GetIndexesProto.Builder builder = GetIndexesProto.newBuilder(); + builder.addAllIndex(store.getAllIndexes()); + return builder.build(); + } catch (Exception e) { + LOG.error("ERROR: cannot get all indexes", e); return null; } finally { rlock.unlock(); @@ -988,18 +1043,6 @@ public BoolProto dropIndex(RpcController controller, IndexNameProto request) return BOOL_TRUE; } - - @Override - public GetIndexesProto getAllIndexes(RpcController controller, NullProto request) throws ServiceException { - rlock.lock(); - try { - return GetIndexesProto.newBuilder().addAllIndex(store.getAllIndexes()).build(); - } catch (Exception e) { - throw new ServiceException(e); - } finally { - rlock.unlock(); - } - } public boolean checkIfBuiltin(FunctionType type) { return type == GENERAL || type == AGGREGATION || type == DISTINCT_AGGREGATION; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/IndexesTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/IndexesTableDescriptor.java index a079a931d5..d527b19d3d 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/IndexesTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/IndexesTableDescriptor.java @@ -27,12 +27,8 @@ class IndexesTableDescriptor extends AbstractTableDescriptor { new ColumnDescriptor("db_id", Type.INT4, 0), new ColumnDescriptor("tid", Type.INT4, 0), new ColumnDescriptor("index_name", Type.TEXT, 0), - new ColumnDescriptor("column_name", Type.TEXT, 0), - new ColumnDescriptor("data_type", Type.TEXT, 0), - new ColumnDescriptor("index_type", Type.TEXT, 0), - new ColumnDescriptor("is_unique", Type.BOOLEAN, 0), - new ColumnDescriptor("is_clustered", Type.BOOLEAN, 0), - new ColumnDescriptor("is_ascending", Type.BOOLEAN, 0) + new ColumnDescriptor("index_method", Type.TEXT, 0), + new ColumnDescriptor("index_path", Type.TEXT, 0), }; public IndexesTableDescriptor(InfoSchemaMetadataDictionary metadataDictionary) { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index b541e67625..0cd880337c 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -27,9 +27,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.tajo.annotation.Nullable; -import org.apache.tajo.catalog.CatalogConstants; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; @@ -1968,37 +1966,55 @@ public void createIndex(final IndexDescProto proto) throws CatalogException { Connection conn = null; PreparedStatement pstmt = null; - String databaseName = proto.getTableIdentifier().getDatabaseName(); - String tableName = proto.getTableIdentifier().getTableName(); - String columnName = CatalogUtil.extractSimpleName(proto.getColumn().getName()); + final String databaseName = proto.getTableIdentifier().getDatabaseName(); + final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName()); try { + // indexes table int databaseId = getDatabaseId(databaseName); int tableId = getTableId(databaseId, databaseName, tableName); String sql = "INSERT INTO " + TB_INDEXES + " (" + COL_DATABASES_PK + ", " + COL_TABLES_PK + ", INDEX_NAME, " + - "COLUMN_NAME, DATA_TYPE, INDEX_TYPE, PATH, IS_UNIQUE, IS_CLUSTERED, IS_ASCENDING) " + - "VALUES (?,?,?,?,?,?,?,?,?,?)"; + "INDEX_TYPE, PATH, COLUMN_NAMES, DATA_TYPES, ORDERS, NULL_ORDERS, IS_UNIQUE, IS_CLUSTERED) " + + "VALUES (?,?,?,?,?,?,?,?,?,?,?)"; if (LOG.isDebugEnabled()) { LOG.debug(sql); } + StringBuilder columnNamesBuilder = new StringBuilder(); + StringBuilder dataTypesBuilder= new StringBuilder(); + StringBuilder ordersBuilder = new StringBuilder(); + StringBuilder nullOrdersBuilder = new StringBuilder(); + for (SortSpecProto columnSpec : proto.getKeySortSpecsList()) { + // Since the key columns are always sorted in order of their occurrence position in the relation schema, + // the concatenated name can be uniquely identified. + columnNamesBuilder.append(columnSpec.getColumn().getName()).append(","); + dataTypesBuilder.append(columnSpec.getColumn().getDataType().getType().name()).append(","); + ordersBuilder.append(columnSpec.getAscending()).append(","); + nullOrdersBuilder.append(columnSpec.getNullFirst()).append(","); + } + columnNamesBuilder.deleteCharAt(columnNamesBuilder.length()-1); + dataTypesBuilder.deleteCharAt(dataTypesBuilder.length()-1); + ordersBuilder.deleteCharAt(ordersBuilder.length()-1); + nullOrdersBuilder.deleteCharAt(nullOrdersBuilder.length()-1); + conn = getConnection(); conn.setAutoCommit(false); pstmt = conn.prepareStatement(sql); pstmt.setInt(1, databaseId); pstmt.setInt(2, tableId); - pstmt.setString(3, proto.getName()); - pstmt.setString(4, columnName); - pstmt.setString(5, proto.getColumn().getDataType().getType().name()); - pstmt.setString(6, proto.getIndexMethod().toString()); - pstmt.setString(7, proto.getIndexPath()); - pstmt.setBoolean(8, proto.hasIsUnique() && proto.getIsUnique()); - pstmt.setBoolean(9, proto.hasIsClustered() && proto.getIsClustered()); - pstmt.setBoolean(10, proto.hasIsAscending() && proto.getIsAscending()); + pstmt.setString(3, proto.getIndexName()); // index name + pstmt.setString(4, proto.getIndexMethod().toString()); // index type + pstmt.setString(5, proto.getIndexPath()); // index path + pstmt.setString(6, columnNamesBuilder.toString()); + pstmt.setString(7, dataTypesBuilder.toString()); + pstmt.setString(8, ordersBuilder.toString()); + pstmt.setString(9, nullOrdersBuilder.toString()); + pstmt.setBoolean(10, proto.hasIsUnique() && proto.getIsUnique()); + pstmt.setBoolean(11, proto.hasIsClustered() && proto.getIsClustered()); pstmt.executeUpdate(); conn.commit(); } catch (SQLException se) { @@ -2052,9 +2068,7 @@ public static String getTableName(Connection conn, int tableId) throws SQLExcept } final static String GET_INDEXES_SQL = - "SELECT " + COL_TABLES_PK + ", INDEX_NAME, COLUMN_NAME, DATA_TYPE, INDEX_TYPE, PATH, IS_UNIQUE, " + - "IS_CLUSTERED, IS_ASCENDING FROM " + TB_INDEXES; - + "SELECT * FROM " + TB_INDEXES; @Override public IndexDescProto getIndexByName(String databaseName, final String indexName) @@ -2085,6 +2099,7 @@ public IndexDescProto getIndexByName(String databaseName, final String indexName resultToIndexDescProtoBuilder(builder, res); String tableName = getTableName(conn, res.getInt(COL_TABLES_PK)); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); + builder.setTargetRelationSchema(getTable(databaseName, tableName).getSchema()); proto = builder.build(); } catch (SQLException se) { throw new CatalogException(se); @@ -2096,9 +2111,8 @@ public IndexDescProto getIndexByName(String databaseName, final String indexName } @Override - public IndexDescProto getIndexByColumn(final String databaseName, - final String tableName, - final String columnName) throws CatalogException { + public IndexDescProto getIndexByColumns(String databaseName, String tableName, String[] columnNames) + throws CatalogException { Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; @@ -2106,25 +2120,35 @@ public IndexDescProto getIndexByColumn(final String databaseName, try { int databaseId = getDatabaseId(databaseName); + int tableId = getTableId(databaseId, databaseName, tableName); + TableDescProto tableDescProto = getTable(databaseName, tableName); - String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND COLUMN_NAME=?"; + String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND " + + COL_TABLES_PK + "=? AND COLUMN_NAMES=?"; if (LOG.isDebugEnabled()) { LOG.debug(sql); } + // Since the column names in the unified name are always sorted + // in order of occurrence position in the relation schema, + // they can be uniquely identified. + String unifiedName = CatalogUtil.buildFQName(databaseName, tableName, + CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames)); conn = getConnection(); pstmt = conn.prepareStatement(sql); pstmt.setInt(1, databaseId); - ; - pstmt.setString(2, columnName); + pstmt.setInt(2, tableId); + pstmt.setString(3, unifiedName); res = pstmt.executeQuery(); if (!res.next()) { - throw new CatalogException("ERROR: there is no index matched to " + columnName); + throw new CatalogException("ERROR: there is no index matched to " + unifiedName); } + IndexDescProto.Builder builder = IndexDescProto.newBuilder(); resultToIndexDescProtoBuilder(builder, res); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); + builder.setTargetRelationSchema(tableDescProto.getSchema()); proto = builder.build(); } catch (SQLException se) { throw new CatalogException(se); @@ -2169,7 +2193,7 @@ public boolean existIndexByName(String databaseName, final String indexName) thr } @Override - public boolean existIndexByColumn(String databaseName, String tableName, String columnName) + public boolean existIndexByColumns(String databaseName, String tableName, String[] columnNames) throws CatalogException { Connection conn = null; ResultSet res = null; @@ -2179,18 +2203,27 @@ public boolean existIndexByColumn(String databaseName, String tableName, String try { int databaseId = getDatabaseId(databaseName); + int tableId = getTableId(databaseId, databaseName, tableName); + Schema relationSchema = new Schema(getTable(databaseName, tableName).getSchema()); String sql = - "SELECT INDEX_NAME FROM " + TB_INDEXES + " WHERE " + COL_DATABASES_PK + "=? AND COLUMN_NAME=?"; + "SELECT " + COL_INDEXES_PK + " FROM " + TB_INDEXES + + " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_PK + "=? AND COLUMN_NAMES=?"; if (LOG.isDebugEnabled()) { LOG.debug(sql); } + // Since the column names in the unified name are always sorted + // in order of occurrence position in the relation schema, + // they can be uniquely identified. + String unifiedName = CatalogUtil.buildFQName(databaseName, tableName, + CatalogUtil.getUnifiedSimpleColumnName(new Schema(relationSchema), columnNames)); conn = getConnection(); pstmt = conn.prepareStatement(sql); pstmt.setInt(1, databaseId); - pstmt.setString(2, columnName); + pstmt.setInt(2, tableId); + pstmt.setString(3, unifiedName); res = pstmt.executeQuery(); exist = res.next(); } catch (SQLException se) { @@ -2202,22 +2235,18 @@ public boolean existIndexByColumn(String databaseName, String tableName, String } @Override - public IndexDescProto[] getIndexes(String databaseName, final String tableName) + public List getAllIndexNamesByTable(final String databaseName, final String tableName) throws CatalogException { - Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; - final List protos = new ArrayList(); + final List indexNames = new ArrayList(); try { final int databaseId = getDatabaseId(databaseName); final int tableId = getTableId(databaseId, databaseName, tableName); - final TableIdentifierProto tableIdentifier = CatalogUtil.buildTableIdentifier(databaseName, tableName); - String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_PK + "=?"; - if (LOG.isDebugEnabled()) { LOG.debug(sql); } @@ -2229,10 +2258,7 @@ public IndexDescProto[] getIndexes(String databaseName, final String tableName) res = pstmt.executeQuery(); while (res.next()) { - IndexDescProto.Builder builder = IndexDescProto.newBuilder(); - resultToIndexDescProtoBuilder(builder, res); - builder.setTableIdentifier(tableIdentifier); - protos.add(builder.build()); + indexNames.add(res.getString("index_name")); } } catch (SQLException se) { throw new CatalogException(se); @@ -2240,57 +2266,73 @@ public IndexDescProto[] getIndexes(String databaseName, final String tableName) CatalogUtil.closeQuietly(pstmt, res); } - return protos.toArray(new IndexDescProto[protos.size()]); + return indexNames; } - - @Override - public List getAllIndexes() throws CatalogException { - Connection conn = null; - Statement stmt = null; - ResultSet resultSet = null; - List indexes = new ArrayList(); + @Override + public boolean existIndexesByTable(String databaseName, String tableName) throws CatalogException { + ResultSet res = null; + PreparedStatement pstmt = null; + final List indexNames = new ArrayList(); try { - String sql = "SELECT " + COL_DATABASES_PK + ", " + COL_TABLES_PK + ", INDEX_NAME, " + - "COLUMN_NAME, DATA_TYPE, INDEX_TYPE, IS_UNIQUE, IS_CLUSTERED, IS_ASCENDING FROM " + TB_INDEXES; + final int databaseId = getDatabaseId(databaseName); + final int tableId = getTableId(databaseId, databaseName, tableName); - conn = getConnection(); - stmt = conn.createStatement(); - resultSet = stmt.executeQuery(sql); - while (resultSet.next()) { - IndexProto.Builder builder = IndexProto.newBuilder(); - - builder.setDbId(resultSet.getInt(COL_DATABASES_PK)); - builder.setTId(resultSet.getInt(COL_TABLES_PK)); - builder.setIndexName(resultSet.getString("INDEX_NAME")); - builder.setColumnName(resultSet.getString("COLUMN_NAME")); - builder.setDataType(resultSet.getString("DATA_TYPE")); - builder.setIndexType(resultSet.getString("INDEX_TYPE")); - builder.setIsUnique(resultSet.getBoolean("IS_UNIQUE")); - builder.setIsClustered(resultSet.getBoolean("IS_CLUSTERED")); - builder.setIsAscending(resultSet.getBoolean("IS_ASCENDING")); - - indexes.add(builder.build()); + String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_PK + "=?"; + + if (LOG.isDebugEnabled()) { + LOG.debug(sql); } + + conn = getConnection(); + pstmt = conn.prepareStatement(sql); + pstmt.setInt(1, databaseId); + pstmt.setInt(2, tableId); + res = pstmt.executeQuery(); + + return res.next(); } catch (SQLException se) { throw new CatalogException(se); } finally { - CatalogUtil.closeQuietly(stmt, resultSet); + CatalogUtil.closeQuietly(pstmt, res); } - - return indexes; + } + + @Override + public List getAllIndexes() throws CatalogException { + List indexDescProtos = TUtil.newList(); + for (String databaseName : getAllDatabaseNames()) { + for (String tableName : getAllTableNames(databaseName)) { + for (String indexName: getAllIndexNamesByTable(databaseName, tableName)) { + indexDescProtos.add(getIndexByName(databaseName, indexName)); + } + } + } + return indexDescProtos; } private void resultToIndexDescProtoBuilder(IndexDescProto.Builder builder, final ResultSet res) throws SQLException { - builder.setName(res.getString("index_name")); - builder.setColumn(indexResultToColumnProto(res)); + builder.setIndexName(res.getString("index_name")); builder.setIndexMethod(getIndexMethod(res.getString("index_type").trim())); builder.setIndexPath(res.getString("path")); + String[] columnNames, dataTypes, orders, nullOrders; + columnNames = res.getString("column_names").trim().split(","); + dataTypes = res.getString("data_types").trim().split(","); + orders = res.getString("orders").trim().split(","); + nullOrders = res.getString("null_orders").trim().split(","); + int columnNum = columnNames.length; + for (int i = 0; i < columnNum; i++) { + SortSpecProto.Builder colSpecBuilder = SortSpecProto.newBuilder(); + colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(columnNames[i]) + .setDataType(CatalogUtil.newSimpleDataType(getDataType(dataTypes[i]))).build()); + colSpecBuilder.setAscending(orders[i].equals("true")); + colSpecBuilder.setNullFirst(nullOrders[i].equals("true")); + builder.addKeySortSpecs(colSpecBuilder.build()); + } builder.setIsUnique(res.getBoolean("is_unique")); builder.setIsClustered(res.getBoolean("is_clustered")); - builder.setIsAscending(res.getBoolean("is_ascending")); } /** diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index ed6fedc790..ed1cbc43c2 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -20,14 +20,7 @@ import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; -import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto; -import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; -import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; +import org.apache.tajo.catalog.proto.CatalogProtos.*; import java.io.Closeable; @@ -36,10 +29,6 @@ import java.util.Collection; import java.util.List; -import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; -import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionMethodProto; -import static org.apache.tajo.catalog.proto.CatalogProtos.TablespaceProto; - public interface CatalogStore extends Closeable { /*************************** Tablespace ******************************/ void createTablespace(String spaceName, String spaceUri) throws CatalogException; @@ -129,18 +118,20 @@ void addPartition(String databaseName, String tableName, void dropIndex(String databaseName, String indexName) throws CatalogException; IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException; - - IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName) + + IndexDescProto getIndexByColumns(String databaseName, String tableName, String[] columnNames) throws CatalogException; boolean existIndexByName(String databaseName, String indexName) throws CatalogException; - - boolean existIndexByColumn(String databaseName, String tableName, String columnName) + + boolean existIndexByColumns(String databaseName, String tableName, String[] columnNames) throws CatalogException; - IndexDescProto [] getIndexes(String databaseName, String tableName) throws CatalogException; - - List getAllIndexes() throws CatalogException; + List getAllIndexNamesByTable(String databaseName, String tableName) throws CatalogException; + + boolean existIndexesByTable(String databaseName, String tableName) throws CatalogException; + + List getAllIndexes() throws CatalogException; /************************** FUNCTION *****************************/ diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 2d35496781..2edb4c42a1 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -26,14 +26,17 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.CatalogConstants; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; -import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto; +import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; @@ -61,7 +64,6 @@ public class MemStore implements CatalogStore { public MemStore(Configuration conf) { } - public void close() throws IOException { databases.clear(); @@ -147,6 +149,8 @@ public void createDatabase(String databaseName, String tablespaceName) throws Ca } databases.put(databaseName, new HashMap()); + indexes.put(databaseName, new HashMap()); + indexesByColumn.put(databaseName, new HashMap()); } @Override @@ -160,6 +164,8 @@ public void dropDatabase(String databaseName) throws CatalogException { throw new NoSuchDatabaseException(databaseName); } databases.remove(databaseName); + indexes.remove(databaseName); + indexesByColumn.remove(databaseName); } @Override @@ -539,17 +545,23 @@ public List getAllPartitions() throws CatalogException { @Override public void createIndex(IndexDescProto proto) throws CatalogException { final String databaseName = proto.getTableIdentifier().getDatabaseName(); + final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName()); Map index = checkAndGetDatabaseNS(indexes, databaseName); Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); + TableDescProto tableDescProto = getTable(databaseName, tableName); - if (index.containsKey(proto.getName())) { - throw new AlreadyExistsIndexException(proto.getName()); + if (index.containsKey(proto.getIndexName())) { + throw new AlreadyExistsIndexException(proto.getIndexName()); } - index.put(proto.getName(), proto); - indexByColumn.put(proto.getTableIdentifier().getTableName() + "." - + CatalogUtil.extractSimpleName(proto.getColumn().getName()), proto); + index.put(proto.getIndexName(), proto); + String originalTableName = proto.getTableIdentifier().getTableName(); + String simpleTableName = CatalogUtil.extractSimpleName(originalTableName); + indexByColumn.put(CatalogUtil.buildFQName(proto.getTableIdentifier().getDatabaseName(), + simpleTableName, + getUnifiedNameForIndexByColumn(proto)), + proto); } /* (non-Javadoc) @@ -558,10 +570,19 @@ public void createIndex(IndexDescProto proto) throws CatalogException { @Override public void dropIndex(String databaseName, String indexName) throws CatalogException { Map index = checkAndGetDatabaseNS(indexes, databaseName); + Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); if (!index.containsKey(indexName)) { throw new NoSuchIndexException(indexName); } + IndexDescProto proto = index.get(indexName); + final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName()); + TableDescProto tableDescProto = getTable(databaseName, tableName); index.remove(indexName); + String originalTableName = proto.getTableIdentifier().getTableName(); + String simpleTableName = CatalogUtil.extractSimpleName(originalTableName); + indexByColumn.remove(CatalogUtil.buildFQName(proto.getTableIdentifier().getDatabaseName(), + simpleTableName, + getUnifiedNameForIndexByColumn(proto))); } /* (non-Javadoc) @@ -577,19 +598,18 @@ public IndexDescProto getIndexByName(String databaseName, String indexName) thro return index.get(indexName); } - /* (non-Javadoc) - * @see CatalogStore#getIndexByName(java.lang.String, java.lang.String) - */ @Override - public IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName) - throws CatalogException { - + public IndexDescProto getIndexByColumns(String databaseName, String tableName, String[] columnNames) throws CatalogException { Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - if (!indexByColumn.containsKey(columnName)) { - throw new NoSuchIndexException(columnName); + String simpleTableName = CatalogUtil.extractSimpleName(tableName); + TableDescProto tableDescProto = getTable(databaseName, simpleTableName); + String qualifiedColumnName = CatalogUtil.buildFQName(databaseName, simpleTableName, + CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames)); + if (!indexByColumn.containsKey(qualifiedColumnName)) { + throw new NoSuchIndexException(qualifiedColumnName); } - return indexByColumn.get(columnName); + return indexByColumn.get(qualifiedColumnName); } @Override @@ -599,50 +619,47 @@ public boolean existIndexByName(String databaseName, String indexName) throws Ca } @Override - public boolean existIndexByColumn(String databaseName, String tableName, String columnName) - throws CatalogException { + public boolean existIndexByColumns(String databaseName, String tableName, String[] columnNames) throws CatalogException { Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - return indexByColumn.containsKey(columnName); + TableDescProto tableDescProto = getTable(databaseName, tableName); + return indexByColumn.containsKey( + CatalogUtil.buildFQName(databaseName, CatalogUtil.extractSimpleName(tableName), + CatalogUtil.getUnifiedSimpleColumnName(new Schema(tableDescProto.getSchema()), columnNames))); } @Override - public IndexDescProto[] getIndexes(String databaseName, String tableName) throws CatalogException { - List protos = new ArrayList(); + public List getAllIndexNamesByTable(String databaseName, String tableName) throws CatalogException { + List indexNames = new ArrayList(); Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); + String simpleTableName = CatalogUtil.extractSimpleName(tableName); for (IndexDescProto proto : indexByColumn.values()) { - if (proto.getTableIdentifier().getTableName().equals(tableName)) { - protos.add(proto); + if (proto.getTableIdentifier().getTableName().equals(simpleTableName)) { + indexNames.add(proto.getIndexName()); } } - return protos.toArray(new IndexDescProto[protos.size()]); + return indexNames; } - + @Override - public List getAllIndexes() throws CatalogException { - List indexList = new ArrayList(); - Set databases = indexes.keySet(); - - for (String databaseName: databases) { - Map indexMap = indexes.get(databaseName); - - for (String indexName: indexMap.keySet()) { - IndexDescProto indexDesc = indexMap.get(indexName); - IndexProto.Builder builder = IndexProto.newBuilder(); - - builder.setColumnName(indexDesc.getColumn().getName()); - builder.setDataType(indexDesc.getColumn().getDataType().getType().toString()); - builder.setIndexName(indexName); - builder.setIndexType(indexDesc.getIndexMethod().toString()); - builder.setIsAscending(indexDesc.hasIsAscending() && indexDesc.getIsAscending()); - builder.setIsClustered(indexDesc.hasIsClustered() && indexDesc.getIsClustered()); - builder.setIsUnique(indexDesc.hasIsUnique() && indexDesc.getIsUnique()); - - indexList.add(builder.build()); + public boolean existIndexesByTable(String databaseName, String tableName) throws CatalogException { + Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); + String simpleTableName = CatalogUtil.extractSimpleName(tableName); + for (IndexDescProto proto : indexByColumn.values()) { + if (proto.getTableIdentifier().getTableName().equals(simpleTableName)) { + return true; } } - - return indexList; + return false; + } + + @Override + public List getAllIndexes() throws CatalogException { + List indexDescProtos = TUtil.newList(); + for (Map indexMap : indexes.values()) { + indexDescProtos.addAll(indexMap.values()); + } + return indexDescProtos; } @Override @@ -666,4 +683,13 @@ public List getAllFunctionNames() throws CatalogException { return null; } + public static String getUnifiedNameForIndexByColumn(IndexDescProto proto) { + StringBuilder sb = new StringBuilder(); + for (SortSpecProto columnSpec : proto.getKeySortSpecsList()) { + String[] identifiers = columnSpec.getColumn().getName().split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP); + sb.append(identifiers[identifiers.length-1]).append("_"); + } + sb.deleteCharAt(sb.length()-1); + return sb.toString(); + } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index b0ba3b994e..5e19ac1bf3 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -97,17 +97,20 @@ @@ -115,7 +118,7 @@ - + @@ -116,7 +119,7 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition. - + indexDescs = TUtil.newHashSet(); + indexDescs.add(desc1); + indexDescs.add(desc2); + indexDescs.add(desc3); + for (IndexDesc index : catalog.getAllIndexesByTable(DEFAULT_DATABASE_NAME, "indexed")) { + assertTrue(indexDescs.contains(index)); + } catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getName()); assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getName())); diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java index db7f981d3d..5ecb1614a5 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java @@ -497,7 +497,7 @@ private void executeJsonQuery(String json) throws ServiceException, IOException if (response == null) { displayFormatter.printErrorMessage(sout, "response is null"); wasError = true; - } else if (response.getResultCode() == ClientProtos.ResultCode.OK) { + } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) { if (response.getIsForwarded()) { QueryId queryId = new QueryId(response.getQueryId()); waitForQueryCompleted(queryId); @@ -510,8 +510,8 @@ private void executeJsonQuery(String json) throws ServiceException, IOException } } } else { - if (response.hasErrorMessage()) { - displayFormatter.printErrorMessage(sout, response.getErrorMessage()); + if (response.getResult().hasErrorMessage()) { + displayFormatter.printErrorMessage(sout, response.getResult().getErrorMessage()); wasError = true; } } @@ -534,7 +534,7 @@ private int executeQuery(String statement) throws ServiceException, IOException if (response == null) { displayFormatter.printErrorMessage(sout, "response is null"); wasError = true; - } else if (response.getResultCode() == ClientProtos.ResultCode.OK) { + } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) { if (response.getIsForwarded()) { QueryId queryId = new QueryId(response.getQueryId()); waitForQueryCompleted(queryId); @@ -546,8 +546,8 @@ private int executeQuery(String statement) throws ServiceException, IOException } } } else { - if (response.hasErrorMessage()) { - displayFormatter.printErrorMessage(sout, response.getErrorMessage()); + if (response.getResult().hasErrorMessage()) { + displayFormatter.printErrorMessage(sout, response.getResult().getErrorMessage()); wasError = true; } } diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java index 5eebc2be01..291561277b 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java @@ -21,9 +21,12 @@ import org.apache.commons.lang.CharUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; import org.apache.tajo.cli.tsql.TajoCli; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.TUtil; @@ -51,6 +54,22 @@ public void invoke(String[] cmd) throws Exception { context.getOutput().println("Did not find any relation named \"" + tableName + "\""); } else { context.getOutput().println(toFormattedString(desc)); + // If there exists any indexes for the table, print index information + if (client.hasIndexes(tableName)) { + StringBuilder sb = new StringBuilder(); + sb.append("Indexes:\n"); + for (IndexDescProto index : client.getIndexes(tableName)) { + sb.append("\"").append(index.getIndexName()).append("\" "); + sb.append(index.getIndexMethod()).append(" ("); + for (SortSpecProto key : index.getKeySortSpecsList()) { + sb.append(CatalogUtil.extractSimpleName(key.getColumn().getName())); + sb.append(key.getAscending() ? " ASC" : " DESC"); + sb.append(key.getNullFirst() ? " NULLS FIRST, " : " NULLS LAST, "); + } + sb.delete(sb.length()-2, sb.length()-1).append(")\n"); + } + context.getOutput().println(sb.toString()); + } } } else if (cmd.length == 1) { List tableList = client.getTableList(null); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java index a36fc0e41e..652008c3db 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClient.java @@ -21,11 +21,13 @@ import com.google.protobuf.ServiceException; import org.apache.hadoop.fs.Path; import org.apache.tajo.annotation.Nullable; +import org.apache.tajo.catalog.IndexMeta; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import java.io.Closeable; import java.sql.SQLException; @@ -134,4 +136,18 @@ public TableDesc createExternalTable(final String tableName, final Schema schema public TableDesc getTableDesc(final String tableName) throws ServiceException; public List getFunctions(final String functionName) throws ServiceException; + + public IndexDescProto getIndex(final String indexName) throws ServiceException; + + public boolean existIndex(final String indexName) throws ServiceException; + + public List getIndexes(final String tableName) throws ServiceException; + + public boolean hasIndexes(final String tableName) throws ServiceException; + + public IndexDescProto getIndex(final String tableName, final String[] columnNames) throws ServiceException; + + public boolean existIndex(final String tableName, final String[] columnName) throws ServiceException; + + public boolean dropIndex(final String indexName) throws ServiceException; } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java index 496161d546..17fdb25da6 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/CatalogAdminClientImpl.java @@ -21,13 +21,12 @@ import com.google.protobuf.ServiceException; import org.apache.hadoop.fs.Path; import org.apache.tajo.annotation.Nullable; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.ipc.ClientProtos; +import org.apache.tajo.ipc.ClientProtos.*; import org.apache.tajo.ipc.TajoMasterClientProtocol; import org.apache.tajo.jdbc.SQLStates; import org.apache.tajo.rpc.NettyClientBase; @@ -37,7 +36,6 @@ import java.sql.SQLException; import java.util.List; -import static org.apache.tajo.ipc.TajoMasterClientProtocol.TajoMasterClientProtocolService; import static org.apache.tajo.ipc.TajoMasterClientProtocol.TajoMasterClientProtocolService.BlockingInterface; public class CatalogAdminClientImpl implements CatalogAdminClient { @@ -152,10 +150,10 @@ public TableDesc call(NettyClientBase client) throws ServiceException, SQLExcept builder.setPartition(partitionMethodDesc.getProto()); } ClientProtos.TableResponse res = tajoMasterService.createExternalTable(null, builder.build()); - if (res.getResultCode() == ClientProtos.ResultCode.OK) { + if (res.getResult().getResultCode() == ClientProtos.ResultCode.OK) { return CatalogUtil.newTableDesc(res.getTableDesc()); } else { - throw new SQLException(res.getErrorMessage(), SQLStates.ER_NO_SUCH_TABLE.getState()); + throw new SQLException(res.getResult().getErrorMessage(), SQLStates.ER_NO_SUCH_TABLE.getState()); } } @@ -226,10 +224,10 @@ public TableDesc call(NettyClientBase client) throws ServiceException, SQLExcept builder.setSessionId(connection.sessionId); builder.setTableName(tableName); ClientProtos.TableResponse res = tajoMasterService.getTableDesc(null, builder.build()); - if (res.getResultCode() == ClientProtos.ResultCode.OK) { + if (res.getResult().getResultCode() == ClientProtos.ResultCode.OK) { return CatalogUtil.newTableDesc(res.getTableDesc()); } else { - throw new SQLException(res.getErrorMessage(), SQLStates.ER_NO_SUCH_TABLE.getState()); + throw new SQLException(res.getResult().getErrorMessage(), SQLStates.ER_NO_SUCH_TABLE.getState()); } } @@ -250,16 +248,134 @@ public List call(NettyClientBase client) throws String paramFunctionName = functionName == null ? "" : functionName; ClientProtos.FunctionResponse res = tajoMasterService.getFunctionList(null, connection.convertSessionedString(paramFunctionName)); - if (res.getResultCode() == ClientProtos.ResultCode.OK) { + if (res.getResult().getResultCode() == ClientProtos.ResultCode.OK) { return res.getFunctionsList(); } else { - throw new SQLException(res.getErrorMessage()); + throw new SQLException(res.getResult().getErrorMessage()); } } }.withRetries(); } + @Override + public IndexDescProto getIndex(final String indexName) throws ServiceException { + return new ServerCallable(connection.connPool, + connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { + + @Override + public IndexDescProto call(NettyClientBase client) throws Exception { + BlockingInterface tajoMasterService = client.getStub(); + return tajoMasterService.getIndexWithName(null, + connection.convertSessionedString(indexName)); + } + }.withRetries(); + } + + @Override + public boolean existIndex(final String indexName) throws ServiceException { + return new ServerCallable(connection.connPool, + connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { + + @Override + public Boolean call(NettyClientBase client) throws Exception { + BlockingInterface tajoMasterService = client.getStub(); + return tajoMasterService.existIndexWithName(null, + connection.convertSessionedString(indexName)).getValue(); + } + }.withRetries(); + } + + @Override + public List getIndexes(final String tableName) throws ServiceException { + return new ServerCallable>(connection.connPool, + connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { + + @Override + public List call(NettyClientBase client) throws Exception { + BlockingInterface tajoMasterService = client.getStub(); + GetIndexesResponse response = tajoMasterService.getIndexesForTable(null, + connection.convertSessionedString(tableName)); + if (response.getResult().getResultCode() == ResultCode.OK) { + return response.getIndexesList(); + } else { + throw new SQLException(response.getResult().getErrorMessage()); + } + } + }.withRetries(); + } + + @Override + public boolean hasIndexes(final String tableName) throws ServiceException { + return new ServerCallable(connection.connPool, + connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { + + @Override + public Boolean call(NettyClientBase client) throws Exception { + BlockingInterface tajoMasterService = client.getStub(); + return tajoMasterService.existIndexesForTable(null, + connection.convertSessionedString(tableName)).getValue(); + } + }.withRetries(); + } + + @Override + public IndexDescProto getIndex(final String tableName, final String[] columnNames) throws ServiceException { + return new ServerCallable(connection.connPool, + connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { + + @Override + public IndexDescProto call(NettyClientBase client) throws Exception { + BlockingInterface tajoMasterService = client.getStub(); + GetIndexWithColumnsRequest.Builder builder = GetIndexWithColumnsRequest.newBuilder(); + builder.setSessionId(connection.sessionId); + builder.setTableName(tableName); + for (String eachColumnName : columnNames) { + builder.addColumnNames(eachColumnName); + } + GetIndexWithColumnsResponse response = tajoMasterService.getIndexWithColumns(null, builder.build()); + if (response.getResult().getResultCode() == ResultCode.OK) { + return response.getIndexDesc(); + } else { + throw new SQLException(response.getResult().getErrorMessage()); + } + } + }.withRetries(); + } + + @Override + public boolean existIndex(final String tableName, final String[] columnName) throws ServiceException { + return new ServerCallable(connection.connPool, + connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { + + @Override + public Boolean call(NettyClientBase client) throws Exception { + BlockingInterface tajoMasterService = client.getStub(); + GetIndexWithColumnsRequest.Builder builder = GetIndexWithColumnsRequest.newBuilder(); + builder.setSessionId(connection.sessionId); + builder.setTableName(tableName); + for (String eachColumnName : columnName) { + builder.addColumnNames(eachColumnName); + } + return tajoMasterService.existIndexWithColumns(null, builder.build()).getValue(); + } + }.withRetries(); + } + + @Override + public boolean dropIndex(final String indexName) throws ServiceException { + return new ServerCallable(connection.connPool, + connection.getTajoMasterAddr(), TajoMasterClientProtocol.class, false, true) { + + @Override + public Boolean call(NettyClientBase client) throws Exception { + BlockingInterface tajoMasterService = client.getStub(); + return tajoMasterService.dropIndex(null, + connection.convertSessionedString(indexName)).getValue(); + } + }.withRetries(); + } + @Override public void close() throws IOException { } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index bab3518cd9..c6436798ad 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -170,7 +170,7 @@ public ClientProtos.SubmitQueryResponse call(NettyClientBase client) throws Serv SubmitQueryResponse response = tajoMasterService.submitQuery(null, builder.build()); - if (response.getResultCode() == ResultCode.OK) { + if (response.getResult().getResultCode() == ResultCode.OK) { connection.updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); } return response; @@ -205,11 +205,11 @@ public ResultSet executeQueryAndGetResult(String sql) throws ServiceException, I ClientProtos.SubmitQueryResponse response = executeQuery(sql); - if (response.getResultCode() == ClientProtos.ResultCode.ERROR) { - if (response.hasErrorMessage()) { - throw new ServiceException(response.getErrorMessage()); - } else if (response.hasErrorTrace()) { - throw new ServiceException(response.getErrorTrace()); + if (response.getResult().getResultCode() == ClientProtos.ResultCode.ERROR) { + if (response.getResult().hasErrorMessage()) { + throw new ServiceException(response.getResult().getErrorMessage()); + } else if (response.getResult().hasErrorTrace()) { + throw new ServiceException(response.getResult().getErrorTrace()); } } @@ -241,8 +241,8 @@ public ResultSet executeJsonQueryAndGetResult(final String json) throws ServiceE ClientProtos.SubmitQueryResponse response = executeQueryWithJson(json); - if (response.getResultCode() == ClientProtos.ResultCode.ERROR) { - throw new ServiceException(response.getErrorTrace()); + if (response.getResult().getResultCode() == ClientProtos.ResultCode.ERROR) { + throw new ServiceException(response.getResult().getErrorTrace()); } QueryId queryId = new QueryId(response.getQueryId()); @@ -391,8 +391,8 @@ public ClientProtos.SerializedResultSet call(NettyClientBase client) throws Serv builder.setFetchRowNum(fetchRowNum); try { GetQueryResultDataResponse response = tajoMasterService.getQueryResultData(null, builder.build()); - if (response.getResultCode() == ClientProtos.ResultCode.ERROR) { - throw new ServiceException(response.getErrorTrace()); + if (response.getResult().getResultCode() == ClientProtos.ResultCode.ERROR) { + throw new ServiceException(response.getResult().getErrorTrace()); } return response.getResultSet(); @@ -434,12 +434,12 @@ public Boolean call(NettyClientBase client) throws ServiceException { builder.setIsJson(false); ClientProtos.UpdateQueryResponse response = tajoMasterService.updateQuery(null, builder.build()); - if (response.getResultCode() == ClientProtos.ResultCode.OK) { + if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) { connection.updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); return true; } else { - if (response.hasErrorMessage()) { - System.err.println("ERROR: " + response.getErrorMessage()); + if (response.getResult().hasErrorMessage()) { + System.err.println("ERROR: " + response.getResult().getErrorMessage()); } return false; } @@ -463,11 +463,11 @@ public Boolean call(NettyClientBase client) throws ServiceException { builder.setQuery(json); builder.setIsJson(true); ClientProtos.UpdateQueryResponse response = tajoMasterService.updateQuery(null, builder.build()); - if (response.getResultCode() == ClientProtos.ResultCode.OK) { + if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) { return true; } else { - if (response.hasErrorMessage()) { - System.err.println("ERROR: " + response.getErrorMessage()); + if (response.getResult().hasErrorMessage()) { + System.err.println("ERROR: " + response.getResult().getErrorMessage()); } return false; } @@ -588,11 +588,11 @@ public QueryInfoProto call(NettyClientBase client) throws ServiceException { TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); GetQueryInfoResponse res = tajoMasterService.getQueryInfo(null,builder.build()); - if (res.getResultCode() == ResultCode.OK) { + if (res.getResult().getResultCode() == ResultCode.OK) { return res.getQueryInfo(); } else { abort(); - throw new ServiceException(res.getErrorMessage()); + throw new ServiceException(res.getResult().getErrorMessage()); } } }.withRetries(); @@ -618,11 +618,11 @@ public QueryHistoryProto call(NettyClientBase client) throws ServiceException { QueryMasterClientProtocolService.BlockingInterface queryMasterService = client.getStub(); GetQueryHistoryResponse res = queryMasterService.getQueryHistory(null,builder.build()); - if (res.getResultCode() == ResultCode.OK) { + if (res.getResult().getResultCode() == ResultCode.OK) { return res.getQueryHistory(); } else { abort(); - throw new ServiceException(res.getErrorMessage()); + throw new ServiceException(res.getResult().getErrorMessage()); } } }.withRetries(); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryStatus.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryStatus.java index 4a38934222..486f95c721 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryStatus.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryStatus.java @@ -41,11 +41,11 @@ public QueryStatus(GetQueryStatusResponse proto) { submitTime = proto.getSubmitTime(); finishTime = proto.getFinishTime(); hasResult = proto.getHasResult(); - if (proto.hasErrorMessage()) { - errorText = proto.getErrorMessage(); + if (proto.getResult().hasErrorMessage()) { + errorText = proto.getResult().getErrorMessage(); } - if (proto.hasErrorTrace()) { - errorTrace = proto.getErrorTrace(); + if (proto.getResult().hasErrorTrace()) { + errorTrace = proto.getResult().getErrorTrace(); } queryMasterHost = proto.getQueryMasterHost(); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 1bc8050625..0f82ae87d6 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -184,11 +184,11 @@ public Map call(NettyClientBase client) throws ServiceException SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request); - if (response.getResultCode() == ResultCode.OK) { + if (response.getResult().getResultCode() == ResultCode.OK) { updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); return Collections.unmodifiableMap(sessionVarsCache); } else { - throw new ServiceException(response.getMessage()); + throw new ServiceException(response.getResult().getErrorMessage()); } } }.withRetries(); @@ -207,11 +207,11 @@ public Map call(NettyClientBase client) throws ServiceException SessionUpdateResponse response = tajoMasterService.updateSessionVariables(null, request); - if (response.getResultCode() == ResultCode.OK) { + if (response.getResult().getResultCode() == ResultCode.OK) { updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); return Collections.unmodifiableMap(sessionVarsCache); } else { - throw new ServiceException(response.getMessage()); + throw new ServiceException(response.getResult().getErrorMessage()); } } }.withRetries(); @@ -334,7 +334,7 @@ protected void checkSessionAndGet(NettyClientBase client) throws ServiceExceptio CreateSessionResponse response = tajoMasterService.createSession(null, builder.build()); - if (response.getResultCode() == ResultCode.OK) { + if (response.getResult().getResultCode() == ResultCode.OK) { sessionId = response.getSessionId(); updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); @@ -343,7 +343,7 @@ protected void checkSessionAndGet(NettyClientBase client) throws ServiceExceptio } } else { - throw new InvalidClientSessionException(response.getMessage()); + throw new InvalidClientSessionException(response.getResult().getErrorMessage()); } } } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java index 8eafc91b9f..952274646b 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClientImpl.java @@ -26,11 +26,13 @@ import org.apache.tajo.QueryId; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.annotation.ThreadSafe; +import org.apache.tajo.catalog.IndexMeta; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.ha.HAServiceUtil; @@ -237,4 +239,39 @@ public TableDesc getTableDesc(final String tableName) throws ServiceException { public List getFunctions(final String functionName) throws ServiceException { return catalogClient.getFunctions(functionName); } + + @Override + public IndexDescProto getIndex(String indexName) throws ServiceException { + return catalogClient.getIndex(indexName); + } + + @Override + public boolean existIndex(String indexName) throws ServiceException { + return catalogClient.existIndex(indexName); + } + + @Override + public List getIndexes(String tableName) throws ServiceException { + return catalogClient.getIndexes(tableName); + } + + @Override + public boolean hasIndexes(String tableName) throws ServiceException { + return catalogClient.hasIndexes(tableName); + } + + @Override + public IndexDescProto getIndex(String tableName, String[] columnNames) throws ServiceException { + return catalogClient.getIndex(tableName, columnNames); + } + + @Override + public boolean existIndex(String tableName, String[] columnName) throws ServiceException { + return catalogClient.existIndex(tableName, columnName); + } + + @Override + public boolean dropIndex(String indexName) throws ServiceException { + return catalogClient.dropIndex(indexName); + } } diff --git a/tajo-client/src/main/proto/ClientProtos.proto b/tajo-client/src/main/proto/ClientProtos.proto index a9f5498bc0..0454901174 100644 --- a/tajo-client/src/main/proto/ClientProtos.proto +++ b/tajo-client/src/main/proto/ClientProtos.proto @@ -31,16 +31,21 @@ enum ResultCode { ERROR = 1; } +message RequestResult { + required ResultCode resultCode = 1; + optional string errorMessage = 2; + optional string errorTrace = 3; +} + message CreateSessionRequest { required string username = 1; optional string baseDatabaseName = 2; } message CreateSessionResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; optional SessionIdProto sessionId = 2; optional KeyValueSetProto sessionVars = 3; - optional string message = 4; } message UpdateSessionVariableRequest { @@ -50,9 +55,8 @@ message UpdateSessionVariableRequest { } message SessionUpdateResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; optional KeyValueSetProto sessionVars = 2; - optional string message = 3; } message SessionedStringProto { @@ -61,9 +65,8 @@ message SessionedStringProto { } message ExplainQueryResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; optional string explain = 2; - optional string errorMessage = 3; } message QueryRequest { @@ -74,9 +77,8 @@ message QueryRequest { } message UpdateQueryResponse { - required ResultCode resultCode = 1; - optional string errorMessage = 2; - optional KeyValueSetProto sessionVars = 3; + required RequestResult result = 1; + optional KeyValueSetProto sessionVars = 2; } message GetQueryResultRequest { @@ -126,7 +128,7 @@ message SerializedResultSet { } message SubmitQueryResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; required QueryIdProto queryId = 2; required string userName = 3; optional bool isForwarded = 4 [default = false]; @@ -138,24 +140,19 @@ message SubmitQueryResponse { optional TableDescProto tableDesc = 8; optional int32 maxRowNum = 9; - optional string errorMessage = 10; - optional string errorTrace = 11; - - optional KeyValueSetProto sessionVars = 12; + optional KeyValueSetProto sessionVars = 10; } message GetQueryStatusResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; required QueryIdProto queryId = 2; optional QueryState state = 3; optional float progress = 4; optional int64 submitTime = 5; optional int64 finishTime = 7; optional bool hasResult = 8; - optional string errorMessage = 9; - optional string errorTrace = 10; - optional string queryMasterHost = 11; - optional int32 queryMasterPort = 12; + optional string queryMasterHost = 9; + optional int32 queryMasterPort = 10; } message GetQueryResultDataRequest { @@ -165,10 +162,8 @@ message GetQueryResultDataRequest { } message GetQueryResultDataResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; required SerializedResultSet resultSet = 2; - optional string errorMessage = 3; - optional string errorTrace = 4; } message GetClusterInfoRequest { @@ -228,15 +223,13 @@ message DropTableRequest { } message TableResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; optional TableDescProto tableDesc = 2; - optional string errorMessage = 3; } message FunctionResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; repeated FunctionDescProto functions = 2; - optional string errorMessage = 3; } message QueryInfoProto { @@ -289,14 +282,32 @@ message QueryHistoryProto { } message GetQueryHistoryResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; optional QueryHistoryProto queryHistory = 2; - optional string errorMessage = 3; } message GetQueryInfoResponse { - required ResultCode resultCode = 1; + required RequestResult result = 1; optional QueryInfoProto queryInfo = 2; - optional string errorMessage = 3; } +message CreateIndexResponse { + required RequestResult result = 1; + optional IndexDescProto indexDesc = 2; +} + +message GetIndexesResponse { + required RequestResult result = 1; + repeated IndexDescProto indexes = 2; +} + +message GetIndexWithColumnsRequest { + required SessionIdProto sessionId = 1; + required string tableName = 2; + repeated string columnNames = 3; +} + +message GetIndexWithColumnsResponse { + required RequestResult result = 1; + optional IndexDescProto indexDesc = 2; +} diff --git a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto index 10ca268540..586f9ab526 100644 --- a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto +++ b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto @@ -68,4 +68,13 @@ service TajoMasterClientProtocolService { rpc getTableList(GetTableListRequest) returns (GetTableListResponse); rpc getTableDesc(GetTableDescRequest) returns (TableResponse); rpc getFunctionList(SessionedStringProto) returns (FunctionResponse); + + // Index Management APIs + rpc getIndexWithName(SessionedStringProto) returns (IndexDescProto); + rpc existIndexWithName(SessionedStringProto) returns (BoolProto); + rpc getIndexesForTable(SessionedStringProto) returns (GetIndexesResponse); + rpc existIndexesForTable(SessionedStringProto) returns (BoolProto); + rpc getIndexWithColumns(GetIndexWithColumnsRequest) returns (GetIndexWithColumnsResponse); + rpc existIndexWithColumns(GetIndexWithColumnsRequest) returns (BoolProto); + rpc dropIndex(SessionedStringProto) returns (BoolProto); } diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java index b7a5da71b3..d65f4c3cfa 100644 --- a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java @@ -171,7 +171,7 @@ public float getFloat(ConfigKey key, Float defaultVal) { } public float getFloat(ConfigKey key) { - return getLong(key, null); + return getFloat(key, null); } public void put(ConfigKey key, String val) { diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index d87bbef0b3..be0639e50a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -123,6 +123,10 @@ public enum SessionVars implements ConfigKey { NULL_CHAR(ConfVars.$TEXT_NULL, "null char of text file output", DEFAULT), CODEGEN(ConfVars.$CODEGEN, "Runtime code generation enabled (experiment)", DEFAULT), + // for index + INDEX_ENABLED(ConfVars.$INDEX_ENABLED, "index scan enabled", DEFAULT), + INDEX_SELECTIVITY_THRESHOLD(ConfVars.$INDEX_SELECTIVITY_THRESHOLD, "the selectivity threshold for index scan", DEFAULT), + // Behavior Control --------------------------------------------------------- ARITHABORT(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT, "If true, a running query will be terminated when an overflow or divide-by-zero occurs.", DEFAULT), diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index ab11ddddef..9a3ec6bfd7 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -319,6 +319,10 @@ public static enum ConfVars implements ConfigKey { $MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite $CODEGEN("tajo.executor.codegen.enabled", false), // Runtime code generation + // for index + $INDEX_ENABLED("tajo.query.index.enabled", false), + $INDEX_SELECTIVITY_THRESHOLD("tajo.query.index.selectivity.threshold", 0.05f), + // Client ----------------------------------------------------------------- $CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour. @@ -498,7 +502,7 @@ public void setIntVar(ConfVars var, int val) { } public static long getLongVar(Configuration conf, ConfVars var) { - assert (var.valClass == Long.class || var.valClass == Integer.class); + assert (var.valClass == Long.class || var.valClass == Integer.class || var.valClass == Float.class); if (var.valClass == Integer.class) { return conf.getInt(var.varname, var.defaultIntVal); } else { diff --git a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java index a1de8602b6..65d795d31d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/TUtil.java @@ -21,6 +21,8 @@ import com.google.common.base.Objects; import java.lang.reflect.Array; +import java.net.URI; +import java.net.URISyntaxException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -299,4 +301,12 @@ public static String getCurrentCodePoint(final int depth) { StackTraceElement element = ste[2 + depth]; return element.getClassName() + ":" + element.getMethodName() + "(" + element.getLineNumber() +")"; } + + public static URI stringToURI(String str) { + try { + return new URI(str); + } catch (URISyntaxException e) { + throw new RuntimeException("Cannot convert " + str + " to the URI type", e); + } + } } diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index e73680d46a..5754040459 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -77,10 +77,19 @@ schema_statement ; index_statement + : create_index_statement + | drop_index_statement + ; + +create_index_statement : CREATE (u=UNIQUE)? INDEX identifier ON table_name (method_specifier)? LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)? ; +drop_index_statement + : DROP INDEX index_name = identifier + ; + database_definition : CREATE DATABASE (if_not_exists)? dbname = identifier ; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java index 79513dc734..4199b000ba 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java @@ -214,4 +214,11 @@ public LogicalNode visitScan(CompilationContext context, LogicalPlan plan, Logic return node; } + + @Override + public LogicalNode visitIndexScan(CompilationContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + IndexScanNode node, Stack stack) throws PlanningException { + visitScan(context, plan, block, node, stack); + return node; + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index d0db2b0dda..aea5a59541 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1187,7 +1187,7 @@ public Expr visitTrim_function(SQLParser.Trim_functionContext ctx) { } @Override - public Expr visitIndex_statement(SQLParser.Index_statementContext ctx) { + public Expr visitCreate_index_statement(SQLParser.Create_index_statementContext ctx) { String indexName = ctx.identifier().getText(); String tableName = ctx.table_name().getText(); Relation relation = new Relation(tableName); @@ -1222,6 +1222,12 @@ public Expr visitIndex_statement(SQLParser.Index_statementContext ctx) { return createIndex; } + @Override + public Expr visitDrop_index_statement(SQLParser.Drop_index_statementContext ctx) { + String indexName = ctx.identifier().getText(); + return new DropIndex(indexName); + } + @Override public Expr visitDatabase_definition(@NotNull SQLParser.Database_definitionContext ctx) { return new CreateDatabase(ctx.identifier().getText(), null, checkIfExist(ctx.if_not_exists())); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 3218e15db9..17a9bb18b9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -30,10 +30,12 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.datum.Datum; import org.apache.tajo.plan.serder.LogicalNodeDeserializer; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; @@ -47,20 +49,18 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.FileUtil; -import org.apache.tajo.util.IndexUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Stack; +import java.util.*; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import static org.apache.tajo.catalog.proto.CatalogProtos.PartitionType; @@ -233,7 +233,7 @@ private PhysicalExec createPlanRecursive(TaskAttemptContext ctx, LogicalNode log return new LimitExec(ctx, limitNode.getInSchema(), limitNode.getOutSchema(), leftExec, limitNode); - case BST_INDEX_SCAN: + case INDEX_SCAN: IndexScanNode indexScanNode = (IndexScanNode) logicalNode; leftExec = createIndexScanExec(ctx, indexScanNode); return leftExec; @@ -1187,15 +1187,8 @@ public PhysicalExec createIndexScanExec(TaskAttemptContext ctx, List fragments = FragmentConvertor.convert(ctx.getConf(), fragmentProtos); - String indexName = IndexUtil.getIndexNameOfFrag(fragments.get(0), annotation.getSortKeys()); - FileStorageManager sm = (FileStorageManager)StorageManager.getFileStorageManager(ctx.getConf()); - Path indexPath = new Path(sm.getTablePath(annotation.getTableName()), "index"); - - TupleComparator comp = new BaseTupleComparator(annotation.getKeySchema(), - annotation.getSortKeys()); - return new BSTIndexScanExec(ctx, annotation, fragments.get(0), new Path(indexPath, indexName), - annotation.getKeySchema(), comp, annotation.getDatum()); - + return new BSTIndexScanExec(ctx, annotation, fragments.get(0), annotation.getIndexPath(), + annotation.getKeySchema(), annotation.getPredicates()); } public static EnforceProperty getAlgorithmEnforceProperty(Enforcer enforcer, LogicalNode node) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 9b37a80764..67f5691ab5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -1524,6 +1524,15 @@ public LogicalNode visitScan(GlobalPlanContext context, LogicalPlan plan, Logica return node; } + @Override + public LogicalNode visitIndexScan(GlobalPlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + IndexScanNode node, Stack stack) throws PlanningException { + ExecutionBlock newBlock = context.plan.newExecutionBlock(); + newBlock.setPlan(node); + context.execBlockMap.put(node.getPID(), newBlock); + return node; + } + @Override public LogicalNode visitPartitionedTableScan(GlobalPlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index 6adc523bbe..3e4c62dede 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -18,23 +18,34 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.hadoop.fs.FileSystem; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; +import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.datum.Datum; import org.apache.tajo.engine.planner.Projector; +import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.EvalNode; -import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.expr.EvalTreeUtil; +import org.apache.tajo.plan.logical.IndexScanNode; +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.index.bst.BSTIndex; +import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; +import java.net.URI; +import java.util.Set; public class BSTIndexScanExec extends PhysicalExec { - private ScanNode scanNode; + private final static Log LOG = LogFactory.getLog(BSTIndexScanExec.class); + private IndexScanNode scanNode; private SeekableScanner fileScanner; private EvalNode qual; @@ -42,31 +53,62 @@ public class BSTIndexScanExec extends PhysicalExec { private Projector projector; - private Datum[] datum = null; - + private Datum[] values = null; + private boolean initialize = true; private float progress; - public BSTIndexScanExec(TaskAttemptContext context, ScanNode scanNode , - FileFragment fragment, Path fileName , Schema keySchema, - TupleComparator comparator , Datum[] datum) throws IOException { + private TableStats inputStats; + + public BSTIndexScanExec(TaskAttemptContext context, + IndexScanNode scanNode , + FileFragment fragment, URI indexPrefix , Schema keySchema, + SimplePredicate [] predicates) throws IOException { super(context, scanNode.getInSchema(), scanNode.getOutSchema()); this.scanNode = scanNode; this.qual = scanNode.getQual(); - this.datum = datum; + + SortSpec[] keySortSpecs = new SortSpec[predicates.length]; + values = new Datum[predicates.length]; + for (int i = 0; i < predicates.length; i++) { + keySortSpecs[i] = predicates[i].getKeySortSpec(); + values[i] = predicates[i].getValue(); + } + + TupleComparator comparator = new BaseTupleComparator(keySchema, + keySortSpecs); + + Schema fileScanOutSchema = mergeSubSchemas(inSchema, keySchema, scanNode.getTargets(), qual); this.fileScanner = StorageManager.getSeekableScanner(context.getConf(), - scanNode.getTableDesc().getMeta(), scanNode.getInSchema(), fragment, outSchema); + scanNode.getTableDesc().getMeta(), inSchema, fragment, fileScanOutSchema); this.fileScanner.init(); this.projector = new Projector(context, inSchema, outSchema, scanNode.getTargets()); - FileSystem fs = fileName.getFileSystem(context.getConf()); - this.reader = new BSTIndex(fs.getConf()). - getIndexReader(fileName, keySchema, comparator); + Path indexPath = new Path(indexPrefix.toString(), context.getUniqueKeyFromFragments()); + this.reader = new BSTIndex(context.getConf()). + getIndexReader(indexPath, keySchema, comparator); this.reader.open(); } + private static Schema mergeSubSchemas(Schema originalSchema, Schema subSchema, Target[] targets, EvalNode qual) { + Schema mergedSchema = new Schema(); + Set qualAndTargets = TUtil.newHashSet(); + qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(qual)); + for (Target target : targets) { + qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(target.getEvalTree())); + } + for (Column column : originalSchema.getColumns()) { + if (subSchema.contains(column) + || qualAndTargets.contains(column) + || qualAndTargets.contains(column)) { + mergedSchema.addColumn(column); + } + } + return mergedSchema; + } + @Override public void init() throws IOException { progress = 0.0f; @@ -76,8 +118,8 @@ public void init() throws IOException { public Tuple next() throws IOException { if(initialize) { //TODO : more complicated condition - Tuple key = new VTuple(datum.length); - key.put(datum); + Tuple key = new VTuple(values.length); + key.put(values); long offset = reader.find(key); if (offset == -1) { reader.close(); @@ -116,8 +158,11 @@ public Tuple next() throws IOException { return outTuple; } else { long offset = reader.next(); - if (offset == -1) return null; + if (offset == -1) { + return null; + } else fileScanner.seek(offset); + return null; } } } @@ -132,6 +177,16 @@ public void rescan() throws IOException { @Override public void close() throws IOException { IOUtils.cleanup(null, reader, fileScanner); + if (fileScanner != null) { + try { + TableStats stats = fileScanner.getInputStats(); + if (stats != null) { + inputStats = (TableStats) stats.clone(); + } + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + } reader = null; fileScanner = null; scanNode = null; @@ -143,4 +198,13 @@ public void close() throws IOException { public float getProgress() { return progress; } + + @Override + public TableStats getInputStats() { + if (fileScanner != null) { + return fileScanner.getInputStats(); + } else { + return inputStats; + } + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java index 72a667dbba..9594b588ac 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java @@ -58,6 +58,7 @@ public Tuple next() throws IOException { } projector.eval(tuple, outTuple); + outTuple.setOffset(tuple.getOffset()); return outTuple; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java index 0592217b5d..f9db842534 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; +import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; @@ -34,6 +35,7 @@ import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; +import java.util.Arrays; public class StoreIndexExec extends UnaryPhysicalExec { private static final Log LOG = LogFactory.getLog(StoreIndexExec.class); @@ -53,7 +55,7 @@ public StoreIndexExec(final TaskAttemptContext context, final CreateIndexNode lo public void init() throws IOException { super.init(); - SortSpec[] sortSpecs = logicalPlan.getSortSpecs(); + SortSpec[] sortSpecs = logicalPlan.getKeySortSpecs(); indexKeys = new int[sortSpecs.length]; keySchema = PlannerUtil.sortSpecsToSchema(sortSpecs); @@ -64,8 +66,7 @@ public void init() throws IOException { } TajoConf conf = context.getConf(); - Path indexPath = new Path(logicalPlan.getIndexPath(), ""+context.getUniqueKeyFromFragments()); - System.out.println("exec: " + indexPath); + Path indexPath = new Path(logicalPlan.getIndexPath().toString(), context.getUniqueKeyFromFragments()); // TODO: Create factory using reflection BSTIndex bst = new BSTIndex(conf); this.comparator = new BaseTupleComparator(keySchema, sortSpecs); @@ -78,17 +79,13 @@ public void init() throws IOException { public Tuple next() throws IOException { Tuple tuple; Tuple keyTuple; - Tuple prevKeyTuple = null; long offset; while((tuple = child.next()) != null) { offset = tuple.getOffset(); keyTuple = new VTuple(keySchema.size()); RowStoreUtil.project(tuple, keyTuple, indexKeys); - if (prevKeyTuple == null || !prevKeyTuple.equals(keyTuple)) { - indexWriter.write(keyTuple, offset); - prevKeyTuple = keyTuple; - } + indexWriter.write(keyTuple, offset); } return null; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java index 29dc845a26..709aa81e03 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java @@ -22,6 +22,7 @@ import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; @SuppressWarnings("unused") public class ErrorInjectionRewriter implements LogicalPlanRewriteRule { @@ -31,12 +32,12 @@ public String getName() { } @Override - public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { + public boolean isEligible(LogicalPlanRewriteRuleContext context) { return true; } @Override - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException { throw new NullPointerException(); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 51964f0fb2..6ba413ca7a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -34,7 +34,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.ipc.ClientProtos; +import org.apache.tajo.ipc.ClientProtos.ResultCode; import org.apache.tajo.master.TajoMaster.MasterContext; import org.apache.tajo.master.exec.DDLExecutor; import org.apache.tajo.master.exec.QueryExecutor; @@ -51,6 +51,7 @@ import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.IPCUtil; import java.io.IOException; import java.sql.SQLException; @@ -90,7 +91,8 @@ public void start() { analyzer = new SQLAnalyzer(); preVerifier = new PreLogicalPlanVerifier(context.getCatalog()); planner = new LogicalPlanner(context.getCatalog()); - optimizer = new LogicalOptimizer(context.getConf()); + // Access path rewriter is enabled only in QueryMasterTask + optimizer = new LogicalOptimizer(context.getConf(), context.getCatalog()); annotatedPlanVerifier = new LogicalPlanVerifier(context.getConf(), context.getCatalog()); } catch (Throwable t) { LOG.error(t.getMessage(), t); @@ -169,13 +171,12 @@ public SubmitQueryResponse executeQuery(Session session, String query, boolean i responseBuilder.setUserName(queryContext.get(SessionVars.USERNAME)); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); responseBuilder.setIsForwarded(true); - responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR); String errorMessage = t.getMessage(); if (t.getMessage() == null) { errorMessage = t.getClass().getName(); } - responseBuilder.setErrorMessage(errorMessage); - responseBuilder.setErrorTrace(StringUtils.stringifyException(t)); + responseBuilder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, + errorMessage, StringUtils.stringifyException(t))); return responseBuilder.build(); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultSystemScanner.java index c6466f5750..9187a695bb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultSystemScanner.java @@ -33,15 +33,7 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; -import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto; -import org.apache.tajo.catalog.proto.CatalogProtos.IndexProto; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; -import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; -import org.apache.tajo.catalog.proto.CatalogProtos.TablespaceProto; +import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.conf.TajoConf; @@ -297,43 +289,35 @@ private List getColumns(Schema outSchema) { return tuples; } - + private List getIndexes(Schema outSchema) { - List indexList = masterContext.getCatalog().getAllIndexes(); + List indexList = masterContext.getCatalog().getAllIndexes(); List tuples = new ArrayList(indexList.size()); List columns = outSchema.getColumns(); Tuple aTuple; - - for (IndexProto index: indexList) { + + for (IndexDescProto index: indexList) { aTuple = new VTuple(outSchema.size()); - + for (int fieldId = 0; fieldId < columns.size(); fieldId++) { Column column = columns.get(fieldId); - + if ("db_id".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createInt4(index.getDbId())); + aTuple.put(fieldId, DatumFactory.createInt4(index.getTableIdentifier().getDbId())); } else if ("tid".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createInt4(index.getTId())); + aTuple.put(fieldId, DatumFactory.createInt4(index.getTableIdentifier().getTid())); } else if ("index_name".equalsIgnoreCase(column.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createText(index.getIndexName())); - } else if ("column_name".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createText(index.getColumnName())); - } else if ("data_type".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createText(index.getDataType())); - } else if ("index_type".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createText(index.getIndexType())); - } else if ("is_unique".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createBool(index.getIsUnique())); - } else if ("is_clustered".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createBool(index.getIsClustered())); - } else if ("is_ascending".equalsIgnoreCase(column.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createBool(index.getIsAscending())); + } else if ("index_method".equalsIgnoreCase(column.getSimpleName())) { + aTuple.put(fieldId, DatumFactory.createText(index.getIndexMethod().name())); + } else if ("index_path".equalsIgnoreCase(column.getSimpleName())) { + aTuple.put(fieldId, DatumFactory.createText(index.getIndexPath())); } } - + tuples.add(aTuple); } - + return tuples; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 249d3353cf..f05f54b559 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -36,6 +36,7 @@ import org.apache.tajo.catalog.exception.NoSuchDatabaseException; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.query.QueryContext; @@ -57,6 +58,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.BoolProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringProto; +import org.apache.tajo.util.IPCUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.NetUtils; import org.apache.tajo.util.ProtoUtil; @@ -143,19 +145,17 @@ public CreateSessionResponse createSession(RpcController controller, CreateSessi String sessionId = context.getSessionManager().createSession(request.getUsername(), databaseName); CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder(); - builder.setResultCode(ResultCode.OK); + builder.setResult(IPCUtil.buildOkRequestResult()); builder.setSessionId(TajoIdProtos.SessionIdProto.newBuilder().setId(sessionId).build()); builder.setSessionVars(ProtoUtil.convertFromMap(context.getSessionManager().getAllVariables(sessionId))); return builder.build(); } catch (NoSuchDatabaseException nsde) { CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder(); - builder.setResultCode(ResultCode.ERROR); - builder.setMessage(nsde.getMessage()); + builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, nsde.getMessage(), null)); return builder.build(); } catch (InvalidSessionException e) { CreateSessionResponse.Builder builder = CreateSessionResponse.newBuilder(); - builder.setResultCode(ResultCode.ERROR); - builder.setMessage(e.getMessage()); + builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, e.getMessage(), null)); return builder.build(); } } @@ -173,15 +173,14 @@ public BoolProto removeSession(RpcController controller, TajoIdProtos.SessionIdP public SessionUpdateResponse buildSessionUpdateOnSuccess(Map variables) { SessionUpdateResponse.Builder builder = SessionUpdateResponse.newBuilder(); - builder.setResultCode(ResultCode.OK); + builder.setResult(IPCUtil.buildOkRequestResult()); builder.setSessionVars(new KeyValueSet(variables).getProto()); return builder.build(); } public SessionUpdateResponse buildSessionUpdateOnError(String message) { SessionUpdateResponse.Builder builder = SessionUpdateResponse.newBuilder(); - builder.setResultCode(ResultCode.ERROR); - builder.setMessage(message); + builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, message, null)); return builder.build(); } @@ -288,12 +287,8 @@ public SubmitQueryResponse submitQuery(RpcController controller, QueryRequest re responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); responseBuilder.setIsForwarded(true); responseBuilder.setUserName(context.getConf().getVar(ConfVars.USERNAME)); - responseBuilder.setResultCode(ResultCode.ERROR); - if (e.getMessage() != null) { - responseBuilder.setErrorMessage(ExceptionUtils.getStackTrace(e)); - } else { - responseBuilder.setErrorMessage("Internal Error"); - } + responseBuilder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, + e.getMessage() == null ? "Internal Error" : ExceptionUtils.getStackTrace(e), null)); return responseBuilder.build(); } } @@ -304,23 +299,15 @@ public UpdateQueryResponse updateQuery(RpcController controller, QueryRequest re try { Session session = context.getSessionManager().getSession(request.getSessionId().getId()); QueryContext queryContext = new QueryContext(conf, session); - if (queryContext.getCurrentDatabase() == null) { - for (Map.Entry e : queryContext.getAllKeyValus().entrySet()) { - System.out.println(e.getKey() + "=" + e.getValue()); - } - } - UpdateQueryResponse.Builder builder = UpdateQueryResponse.newBuilder(); + UpdateQueryResponse.Builder responseBuilder = UpdateQueryResponse.newBuilder(); try { context.getGlobalEngine().updateQuery(queryContext, request.getQuery(), request.getIsJson()); - builder.setResultCode(ResultCode.OK); - return builder.build(); + return responseBuilder.setResult(IPCUtil.buildOkRequestResult()).build(); } catch (Exception e) { - builder.setResultCode(ResultCode.ERROR); - if (e.getMessage() == null) { - builder.setErrorMessage(ExceptionUtils.getStackTrace(e)); - } - return builder.build(); + responseBuilder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, + e.getMessage() == null ? ExceptionUtils.getStackTrace(e) : null, null)); + return responseBuilder.build(); } } catch (Throwable t) { throw new ServiceException(t); @@ -460,7 +447,7 @@ public GetQueryStatusResponse getQueryStatus(RpcController controller, builder.setQueryId(request.getQueryId()); if (queryId.equals(QueryIdFactory.NULL_QUERY_ID)) { - builder.setResultCode(ResultCode.OK); + builder.setResult(IPCUtil.buildOkRequestResult()); builder.setState(TajoProtos.QueryState.QUERY_SUCCEEDED); } else { QueryInProgress queryInProgress = context.getQueryJobManager().getQueryInProgress(queryId); @@ -474,7 +461,7 @@ public GetQueryStatusResponse getQueryStatus(RpcController controller, } if (queryInfo != null) { - builder.setResultCode(ResultCode.OK); + builder.setResult(IPCUtil.buildOkRequestResult()); builder.setState(queryInfo.getQueryState()); boolean isCreateTable = queryInfo.getQueryContext().isCreateTable(); @@ -495,11 +482,11 @@ public GetQueryStatusResponse getQueryStatus(RpcController controller, } else { Session session = context.getSessionManager().getSession(request.getSessionId().getId()); if (session.getNonForwardQueryResultScanner(queryId) != null) { - builder.setResultCode(ResultCode.OK); + builder.setResult(IPCUtil.buildOkRequestResult()); builder.setState(TajoProtos.QueryState.QUERY_SUCCEEDED); } else { - builder.setResultCode(ResultCode.ERROR); - builder.setErrorMessage("No such query: " + queryId.toString()); + builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, + "No such query: " + queryId.toString(), null)); } } } @@ -531,17 +518,16 @@ public GetQueryResultDataResponse getQueryResultData(RpcController controller, G resultSetBuilder.addAllSerializedTuples(rows); builder.setResultSet(resultSetBuilder.build()); - builder.setResultCode(ResultCode.OK); + builder.setResult(IPCUtil.buildOkRequestResult()); LOG.info("Send result to client for " + request.getSessionId().getId() + "," + queryId + ", " + rows.size() + " rows"); } catch (Throwable t) { LOG.error(t.getMessage(), t); - builder.setResultCode(ResultCode.ERROR); String errorMessage = t.getMessage() == null ? t.getClass().getName() : t.getMessage(); - builder.setErrorMessage(errorMessage); - builder.setErrorTrace(org.apache.hadoop.util.StringUtils.stringifyException(t)); + builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, + errorMessage, org.apache.hadoop.util.StringUtils.stringifyException(t))); } return builder.build(); } @@ -581,11 +567,11 @@ public GetQueryInfoResponse getQueryInfo(RpcController controller, QueryIdReques if (queryInfo != null) { builder.setQueryInfo(queryInfo.getProto()); } - builder.setResultCode(ResultCode.OK); + builder.setResult(IPCUtil.buildOkRequestResult()); } catch (Throwable t) { LOG.warn(t.getMessage(), t); - builder.setResultCode(ResultCode.ERROR); - builder.setErrorMessage(org.apache.hadoop.util.StringUtils.stringifyException(t)); + builder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, + org.apache.hadoop.util.StringUtils.stringifyException(t), null)); } return builder.build(); @@ -779,13 +765,13 @@ public TableResponse getTableDesc(RpcController controller, GetTableDescRequest if (catalog.existsTable(databaseName, tableName)) { return TableResponse.newBuilder() - .setResultCode(ResultCode.OK) + .setResult(IPCUtil.buildOkRequestResult()) .setTableDesc(catalog.getTableDesc(databaseName, tableName).getProto()) .build(); } else { return TableResponse.newBuilder() - .setResultCode(ResultCode.ERROR) - .setErrorMessage("ERROR: no such a table: " + request.getTableName()) + .setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, + "ERROR: no such a table: " + request.getTableName(), null)) .build(); } } catch (Throwable t) { @@ -821,21 +807,21 @@ public TableResponse createExternalTable(RpcController controller, CreateTableRe meta, path, true, partitionDesc, false); } catch (Exception e) { return TableResponse.newBuilder() - .setResultCode(ResultCode.ERROR) - .setErrorMessage(e.getMessage()).build(); + .setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, e.getMessage(), null)) + .build(); } return TableResponse.newBuilder() - .setResultCode(ResultCode.OK) + .setResult(IPCUtil.buildOkRequestResult()) .setTableDesc(desc.getProto()).build(); } catch (InvalidSessionException ise) { return TableResponse.newBuilder() - .setResultCode(ResultCode.ERROR) - .setErrorMessage(ise.getMessage()).build(); + .setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, ise.getMessage(), null)) + .build(); } catch (IOException ioe) { return TableResponse.newBuilder() - .setResultCode(ResultCode.ERROR) - .setErrorMessage(ioe.getMessage()).build(); + .setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, ioe.getMessage(), null)) + .build(); } } @@ -875,12 +861,184 @@ public FunctionResponse getFunctionList(RpcController controller, SessionedStrin } } return FunctionResponse.newBuilder() - .setResultCode(ResultCode.OK) + .setResult(IPCUtil.buildOkRequestResult()) .addAllFunctions(functionProtos) .build(); } catch (Throwable t) { throw new ServiceException(t); } } + + @Override + public IndexDescProto getIndexWithName(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String indexName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + indexName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + indexName = request.getValue(); + } + return catalog.getIndexByName(databaseName, indexName).getProto(); + } catch (Throwable t) { + throw new ServiceException(t); + } + } + + @Override + public BoolProto existIndexWithName(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String indexName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + indexName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + indexName = request.getValue(); + } + return catalog.existIndexByName(databaseName, indexName) ? + ProtoUtil.TRUE : ProtoUtil.FALSE; + } catch (Throwable t) { + throw new ServiceException(t); + } + } + + @Override + public GetIndexesResponse getIndexesForTable(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String tableName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + tableName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + tableName = request.getValue(); + } + + GetIndexesResponse.Builder builder = GetIndexesResponse.newBuilder(); + for (IndexDesc index : catalog.getAllIndexesByTable(databaseName, tableName)) { + builder.addIndexes(index.getProto()); + } + builder.setResult(IPCUtil.buildOkRequestResult()); + return builder.build(); + } catch (Throwable t) { + throw new ServiceException(t); + } + } + + @Override + public BoolProto existIndexesForTable(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String tableName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + tableName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + tableName = request.getValue(); + } + return catalog.existIndexesByTable(databaseName, tableName) ? + ProtoUtil.TRUE : ProtoUtil.FALSE; + } catch (Throwable t) { + throw new ServiceException(t); + } + } + + @Override + public GetIndexWithColumnsResponse getIndexWithColumns(RpcController controller, GetIndexWithColumnsRequest request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String tableName, databaseName; + if (CatalogUtil.isFQTableName(request.getTableName())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getTableName()); + databaseName = splitted[0]; + tableName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + tableName = request.getTableName(); + } + String[] columnNames = new String[request.getColumnNamesCount()]; + columnNames = request.getColumnNamesList().toArray(columnNames); + + GetIndexWithColumnsResponse.Builder builder = GetIndexWithColumnsResponse.newBuilder(); + builder.setResult(IPCUtil.buildOkRequestResult()); + builder.setIndexDesc(catalog.getIndexByColumnNames(databaseName, tableName, columnNames).getProto()); + return builder.build(); + } catch (Throwable t) { + throw new ServiceException(t); + } + } + + @Override + public BoolProto existIndexWithColumns(RpcController controller, GetIndexWithColumnsRequest request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String tableName, databaseName; + if (CatalogUtil.isFQTableName(request.getTableName())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getTableName()); + databaseName = splitted[0]; + tableName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + tableName = request.getTableName(); + } + String[] columnNames = new String[request.getColumnNamesCount()]; + columnNames = request.getColumnNamesList().toArray(columnNames); + return catalog.existIndexByColumnNames(databaseName, tableName, columnNames) ? + ProtoUtil.TRUE : ProtoUtil.FALSE; + } catch (Throwable t) { + throw new ServiceException(t); + } + } + + @Override + public BoolProto dropIndex(RpcController controller, SessionedStringProto request) + throws ServiceException { + try { + context.getSessionManager().touch(request.getSessionId().getId()); + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String indexName, databaseName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + indexName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + indexName = request.getValue(); + } + return catalog.dropIndex(databaseName, indexName) ? + ProtoUtil.TRUE : ProtoUtil.FALSE; + } catch (Throwable t) { + throw new ServiceException(t); + } + } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index acbaa0123e..5b12438e37 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -68,45 +68,88 @@ public boolean execute(QueryContext queryContext, LogicalPlan plan) throws IOExc switch (root.getType()) { - case ALTER_TABLESPACE: - AlterTablespaceNode alterTablespace = (AlterTablespaceNode) root; - alterTablespace(context, queryContext, alterTablespace); - return true; - - - case CREATE_DATABASE: - CreateDatabaseNode createDatabase = (CreateDatabaseNode) root; - createDatabase(queryContext, createDatabase.getDatabaseName(), null, createDatabase.isIfNotExists()); - return true; - case DROP_DATABASE: - DropDatabaseNode dropDatabaseNode = (DropDatabaseNode) root; - dropDatabase(queryContext, dropDatabaseNode.getDatabaseName(), dropDatabaseNode.isIfExists()); - return true; - - - case CREATE_TABLE: - CreateTableNode createTable = (CreateTableNode) root; - createTable(queryContext, createTable, createTable.isIfNotExists()); - return true; - case DROP_TABLE: - DropTableNode dropTable = (DropTableNode) root; - dropTable(queryContext, dropTable.getTableName(), dropTable.isIfExists(), dropTable.isPurge()); - return true; - case TRUNCATE_TABLE: - TruncateTableNode truncateTable = (TruncateTableNode) root; - truncateTable(queryContext, truncateTable); - return true; - - case ALTER_TABLE: - AlterTableNode alterTable = (AlterTableNode) root; - alterTable(context, queryContext, alterTable); - return true; + case ALTER_TABLESPACE: + AlterTablespaceNode alterTablespace = (AlterTablespaceNode) root; + alterTablespace(context, queryContext, alterTablespace); + return true; + + + case CREATE_DATABASE: + CreateDatabaseNode createDatabase = (CreateDatabaseNode) root; + createDatabase(queryContext, createDatabase.getDatabaseName(), null, createDatabase.isIfNotExists()); + return true; + case DROP_DATABASE: + DropDatabaseNode dropDatabaseNode = (DropDatabaseNode) root; + dropDatabase(queryContext, dropDatabaseNode.getDatabaseName(), dropDatabaseNode.isIfExists()); + return true; + + + case CREATE_TABLE: + CreateTableNode createTable = (CreateTableNode) root; + createTable(queryContext, createTable, createTable.isIfNotExists()); + return true; + case DROP_TABLE: + DropTableNode dropTable = (DropTableNode) root; + dropTable(queryContext, dropTable.getTableName(), dropTable.isIfExists(), dropTable.isPurge()); + return true; + case TRUNCATE_TABLE: + TruncateTableNode truncateTable = (TruncateTableNode) root; + truncateTable(queryContext, truncateTable); + return true; + + case ALTER_TABLE: + AlterTableNode alterTable = (AlterTableNode) root; + alterTable(context, queryContext, alterTable); + return true; + + case CREATE_INDEX: + // The catalog information for the created index is automatically updated when the query is successfully finished. + // See the Query.CreateIndexHook class. + return true; + + case DROP_INDEX: + DropIndexNode dropIndexNode = (DropIndexNode) root; + dropIndex(queryContext, dropIndexNode); + return true; default: throw new InternalError("updateQuery cannot handle such query: \n" + root.toJson()); } } + public void dropIndex(final QueryContext queryContext, final DropIndexNode dropIndexNode) { + String databaseName, simpleIndexName; + if (CatalogUtil.isFQTableName(dropIndexNode.getIndexName())) { + String [] splits = CatalogUtil.splitFQTableName(dropIndexNode.getIndexName()); + databaseName = splits[0]; + simpleIndexName = splits[1]; + } else { + databaseName = queryContext.getCurrentDatabase(); + simpleIndexName = dropIndexNode.getIndexName(); + } + + if (!catalog.existIndexByName(databaseName, simpleIndexName)) { + throw new NoSuchIndexException(simpleIndexName); + } + + IndexDesc desc = catalog.getIndexByName(databaseName, simpleIndexName); + + if (!catalog.dropIndex(databaseName, simpleIndexName)) { + LOG.info("Cannot drop index \"" + simpleIndexName + "\"."); + throw new CatalogException("Cannot drop index \"" + simpleIndexName + "\"."); + } + + Path indexPath = new Path(desc.getIndexPath()); + try { + FileSystem fs = indexPath.getFileSystem(context.getConf()); + fs.delete(indexPath, true); + } catch (IOException e) { + throw new InternalError(e.getMessage()); + } + + LOG.info("Index " + simpleIndexName + " is dropped."); + } + /** * Alter a given table */ diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index 22424454ab..26476a3a02 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -29,8 +29,10 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.exception.AlreadyExistsIndexException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; @@ -40,6 +42,7 @@ import org.apache.tajo.engine.planner.physical.StoreTableExec; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.ClientProtos; +import org.apache.tajo.ipc.ClientProtos.ResultCode; import org.apache.tajo.ipc.ClientProtos.SubmitQueryResponse; import org.apache.tajo.master.NonForwardQueryResultFileScanner; import org.apache.tajo.master.NonForwardQueryResultScanner; @@ -58,6 +61,7 @@ import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.storage.*; +import org.apache.tajo.util.IPCUtil; import org.apache.tajo.util.ProtoUtil; import org.apache.tajo.worker.TaskAttemptContext; @@ -98,10 +102,18 @@ public SubmitQueryResponse execute(QueryContext queryContext, Session session, S } else if (PlannerUtil.checkIfDDLPlan(rootNode)) { context.getSystemMetrics().counter("Query", "numDDLQuery").inc(); - ddlExecutor.execute(queryContext, plan); - response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - response.setResultCode(ClientProtos.ResultCode.OK); + if (PlannerUtil.isDistExecDDL(rootNode)) { + if (rootNode.getChild().getType() == NodeType.CREATE_INDEX) { + checkIndexExistence(queryContext, (CreateIndexNode) rootNode.getChild()); + } + executeDistributedQuery(queryContext, session, plan, sql, jsonExpr, response); + } else { + response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); + response.setResult(IPCUtil.buildOkRequestResult()); + } + + ddlExecutor.execute(queryContext, plan); } else if (plan.isExplain()) { // explain query execExplain(plan, response); @@ -142,8 +154,8 @@ public void execSetSession(Session session, LogicalPlan plan, session.selectDatabase(setSessionNode.getValue()); } else { response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - response.setResultCode(ClientProtos.ResultCode.ERROR); - response.setErrorMessage("database \"" + databaseName + "\" does not exists."); + response.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, + "database \"" + databaseName + "\" does not exists.", null)); } // others @@ -157,7 +169,7 @@ public void execSetSession(Session session, LogicalPlan plan, context.getSystemMetrics().counter("Query", "numDDLQuery").inc(); response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - response.setResultCode(ClientProtos.ResultCode.OK); + response.setResult(IPCUtil.buildOkRequestResult()); } public void execExplain(LogicalPlan plan, SubmitQueryResponse.Builder response) throws IOException { @@ -183,7 +195,7 @@ public void execExplain(LogicalPlan plan, SubmitQueryResponse.Builder response) response.setResultSet(serializedResBuilder.build()); response.setMaxRowNum(lines.length); - response.setResultCode(ClientProtos.ResultCode.OK); + response.setResult(IPCUtil.buildOkRequestResult()); response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); } @@ -205,7 +217,7 @@ public void execQueryOnVirtualTable(QueryContext queryContext, Session session, response.setQueryId(queryId.getProto()); response.setMaxRowNum(maxRow); response.setTableDesc(queryResultScanner.getTableDesc().getProto()); - response.setResultCode(ClientProtos.ResultCode.OK); + response.setResult(IPCUtil.buildOkRequestResult()); } public void execSimpleQuery(QueryContext queryContext, Session session, String query, LogicalPlan plan, @@ -235,7 +247,7 @@ public void execSimpleQuery(QueryContext queryContext, Session session, String q response.setQueryId(queryId.getProto()); response.setMaxRowNum(maxRow); response.setTableDesc(desc.getProto()); - response.setResultCode(ClientProtos.ResultCode.OK); + response.setResult(IPCUtil.buildOkRequestResult()); } public void execNonFromQuery(QueryContext queryContext, Session session, String query, @@ -267,7 +279,7 @@ public void execNonFromQuery(QueryContext queryContext, Session session, String responseBuilder.setResultSet(serializedResBuilder); responseBuilder.setMaxRowNum(1); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - responseBuilder.setResultCode(ClientProtos.ResultCode.OK); + responseBuilder.setResult(IPCUtil.buildOkRequestResult()); } } @@ -369,7 +381,7 @@ private void insertNonFromQuery(QueryContext queryContext, // If queryId == NULL_QUERY_ID and MaxRowNum == -1, TajoCli prints only number of inserted rows. responseBuilder.setMaxRowNum(-1); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - responseBuilder.setResultCode(ClientProtos.ResultCode.OK); + responseBuilder.setResult(IPCUtil.buildOkRequestResult()); } public void executeDistributedQuery(QueryContext queryContext, Session session, @@ -398,13 +410,13 @@ public void executeDistributedQuery(QueryContext queryContext, Session session, if(queryInfo == null) { responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR); - responseBuilder.setErrorMessage("Fail starting QueryMaster."); + responseBuilder.setResult(IPCUtil.buildRequestResult(ResultCode.ERROR, + "Fail starting QueryMaster.", null)); LOG.error("Fail starting QueryMaster: " + sql); } else { responseBuilder.setIsForwarded(true); responseBuilder.setQueryId(queryInfo.getQueryId().getProto()); - responseBuilder.setResultCode(ClientProtos.ResultCode.OK); + responseBuilder.setResult(IPCUtil.buildOkRequestResult()); if(queryInfo.getQueryMasterHost() != null) { responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost()); } @@ -413,4 +425,23 @@ public void executeDistributedQuery(QueryContext queryContext, Session session, " is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort()); } } + + private void checkIndexExistence(final QueryContext queryContext, final CreateIndexNode createIndexNode) + throws IOException { + String databaseName, simpleIndexName, qualifiedIndexName; + if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) { + String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName()); + databaseName = splits[0]; + simpleIndexName = splits[1]; + qualifiedIndexName = createIndexNode.getIndexName(); + } else { + databaseName = queryContext.getCurrentDatabase(); + simpleIndexName = createIndexNode.getIndexName(); + qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName); + } + + if (catalog.existIndexByName(databaseName, simpleIndexName)) { + throw new AlreadyExistsIndexException(qualifiedIndexName); + } + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index a626df1c05..88ecebc8fe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -32,6 +32,8 @@ import org.apache.tajo.QueryId; import org.apache.tajo.SessionVars; import org.apache.tajo.TajoProtos.QueryState; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.TableDesc; @@ -472,6 +474,7 @@ public QueryHookExecutor(QueryMaster.QueryMasterContext context) { hookList.add(new MaterializedResultHook()); hookList.add(new CreateTableHook()); hookList.add(new InsertTableHook()); + hookList.add(new CreateIndexHook()); } public void execute(QueryContext queryContext, Query query, @@ -485,6 +488,48 @@ public void execute(QueryContext queryContext, Query query, } } + private class CreateIndexHook implements QueryHook { + + @Override + public boolean isEligible(QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) { + Stage lastStage = query.getStage(finalExecBlockId); + return lastStage.getBlock().getPlan().getType() == NodeType.CREATE_INDEX; + } + + @Override + public void execute(QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { + CatalogService catalog = context.getWorkerContext().getCatalog(); + Stage lastStage = query.getStage(finalExecBlockId); + + CreateIndexNode createIndexNode = (CreateIndexNode) lastStage.getBlock().getPlan(); + String databaseName, simpleIndexName, qualifiedIndexName; + if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) { + String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName()); + databaseName = splits[0]; + simpleIndexName = splits[1]; + qualifiedIndexName = createIndexNode.getIndexName(); + } else { + databaseName = queryContext.getCurrentDatabase(); + simpleIndexName = createIndexNode.getIndexName(); + qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName); + } + ScanNode scanNode = PlannerUtil.findTopNode(createIndexNode, NodeType.SCAN); + if (scanNode == null) { + throw new IOException("Cannot find the table of the relation"); + } + IndexDesc indexDesc = new IndexDesc(databaseName, scanNode.getTableName(), + simpleIndexName, createIndexNode.getIndexPath(), + createIndexNode.getKeySortSpecs(), createIndexNode.getIndexMethod(), + createIndexNode.isUnique(), false, scanNode.getLogicalSchema()); + if (catalog.createIndex(indexDesc)) { + LOG.info("Index " + qualifiedIndexName + " is created for the table " + scanNode.getTableName() + "."); + } else { + LOG.info("Index creation " + qualifiedIndexName + " is failed."); + throw new CatalogException("Cannot create index \"" + qualifiedIndexName + "\"."); + } + } + } + private class MaterializedResultHook implements QueryHook { @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index 9c789a544f..a125415574 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -38,6 +38,12 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.logical.LogicalRootNode; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.UnimplementedException; @@ -48,15 +54,9 @@ import org.apache.tajo.master.event.*; import org.apache.tajo.master.rm.TajoWorkerResourceManager; import org.apache.tajo.master.session.Session; -import org.apache.tajo.plan.LogicalOptimizer; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.LogicalRootNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; -import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.RpcConnectionPool; @@ -347,9 +347,10 @@ public synchronized void startQuery() { LOG.warn("Query already started"); return; } + LOG.info(SessionVars.INDEX_ENABLED.keyname() + " : " + queryContext.getBool(SessionVars.INDEX_ENABLED)); CatalogService catalog = getQueryTaskContext().getQueryMasterContext().getWorkerContext().getCatalog(); LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(systemConf); + LogicalOptimizer optimizer = new LogicalOptimizer(systemConf, catalog); Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class); jsonExpr = null; // remove the possible OOM plan = planner.createPlan(queryContext, expr); @@ -393,6 +394,14 @@ public synchronized void startQuery() { tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); } } + + scanNodes = PlannerUtil.findAllNodes(block.getRoot(), NodeType.INDEX_SCAN); + if (scanNodes != null) { + for (LogicalNode eachScanNode : scanNodes) { + ScanNode scanNode = (ScanNode) eachScanNode; + tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); + } + } } MasterPlan masterPlan = new MasterPlan(queryId, queryContext, plan); queryMasterContext.getGlobalPlanner().build(masterPlan); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/IPCUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/IPCUtil.java new file mode 100644 index 0000000000..dcffe62691 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/util/IPCUtil.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.util; + +import org.apache.tajo.annotation.Nullable; +import org.apache.tajo.ipc.ClientProtos.RequestResult; +import org.apache.tajo.ipc.ClientProtos.ResultCode; + +public class IPCUtil { + + public static RequestResult buildOkRequestResult() { + return buildRequestResult(ResultCode.OK, null, null); + } + + public static RequestResult buildRequestResult(ResultCode code, + @Nullable String errorMessage, + @Nullable String errorTrace) { + RequestResult.Builder builder = RequestResult.newBuilder(); + builder.setResultCode(code); + if (errorMessage != null) { + builder.setErrorMessage(errorMessage); + } + if (errorTrace != null) { + builder.setErrorTrace(errorTrace); + } + return builder.build(); + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java deleted file mode 100644 index 3147bb60e7..0000000000 --- a/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.util; - -import com.google.gson.Gson; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.expr.*; -import org.apache.tajo.plan.logical.IndexScanNode; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.storage.fragment.FileFragment; - -import java.util.Iterator; -import java.util.LinkedList; -import java.util.Map.Entry; - -public class IndexUtil { - public static String getIndexNameOfFrag(FileFragment fragment, SortSpec[] keys) { - StringBuilder builder = new StringBuilder(); - builder.append(fragment.getPath().getName() + "_"); - builder.append(fragment.getStartKey() + "_" + fragment.getLength() + "_"); - for(int i = 0 ; i < keys.length ; i ++) { - builder.append(keys[i].getSortKey().getSimpleName()+"_"); - } - builder.append("_index"); - return builder.toString(); - - } - - public static String getIndexName(String indexName , SortSpec[] keys) { - StringBuilder builder = new StringBuilder(); - builder.append(indexName + "_"); - for(int i = 0 ; i < keys.length ; i ++) { - builder.append(keys[i].getSortKey().getSimpleName() + "_"); - } - return builder.toString(); - } - - public static IndexScanNode indexEval(LogicalPlan plan, ScanNode scanNode, - Iterator> iter ) { - - EvalNode qual = scanNode.getQual(); - Gson gson = CoreGsonHelper.getInstance(); - - FieldAndValueFinder nodeFinder = new FieldAndValueFinder(); - qual.preOrder(nodeFinder); - LinkedList nodeList = nodeFinder.getNodeList(); - - int maxSize = Integer.MIN_VALUE; - SortSpec[] maxIndex = null; - - String json; - while(iter.hasNext()) { - Entry entry = iter.next(); - json = entry.getValue(); - SortSpec[] sortKey = gson.fromJson(json, SortSpec[].class); - if(sortKey.length > nodeList.size()) { - /* If the number of the sort key is greater than where condition, - * this index cannot be used - * */ - continue; - } else { - boolean[] equal = new boolean[sortKey.length]; - for(int i = 0 ; i < sortKey.length ; i ++) { - for(int j = 0 ; j < nodeList.size() ; j ++) { - Column col = ((FieldEval)(nodeList.get(j).getLeftExpr())).getColumnRef(); - if(col.equals(sortKey[i].getSortKey())) { - equal[i] = true; - } - } - } - boolean chk = true; - for(int i = 0 ; i < equal.length ; i ++) { - chk = chk && equal[i]; - } - if(chk) { - if(maxSize < sortKey.length) { - maxSize = sortKey.length; - maxIndex = sortKey; - } - } - } - } - if(maxIndex == null) { - return null; - } else { - Schema keySchema = new Schema(); - for(int i = 0 ; i < maxIndex.length ; i ++ ) { - keySchema.addColumn(maxIndex[i].getSortKey()); - } - Datum[] datum = new Datum[nodeList.size()]; - for(int i = 0 ; i < nodeList.size() ; i ++ ) { - datum[i] = ((ConstEval)(nodeList.get(i).getRightExpr())).getValue(); - } - - return new IndexScanNode(plan.newPID(), scanNode, keySchema , datum , maxIndex); - } - - } - - - private static class FieldAndValueFinder implements EvalNodeVisitor { - private LinkedList nodeList = new LinkedList(); - - public LinkedList getNodeList () { - return this.nodeList; - } - - @Override - public void visit(EvalNode node) { - BinaryEval binaryEval = (BinaryEval) node; - switch(node.getType()) { - case AND: - break; - case EQUAL: - if( binaryEval.getLeftExpr().getType() == EvalType.FIELD - && binaryEval.getRightExpr().getType() == EvalType.CONST ) { - nodeList.add(binaryEval); - } - break; - case IS_NULL: - if( binaryEval.getLeftExpr().getType() == EvalType.FIELD - && binaryEval.getRightExpr().getType() == EvalType.CONST) { - nodeList.add(binaryEval); - } - } - } - } -} diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index da25fe6515..cd9e6ef474 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -304,7 +304,7 @@ public void run() { LOG.error("Internal Error: SubmissionResponse is NULL"); error = new Exception("Internal Error: SubmissionResponse is NULL"); - } else if (response.getResultCode() == ClientProtos.ResultCode.OK) { + } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.OK) { if (response.getIsForwarded()) { queryId = new QueryId(response.getQueryId()); getQueryResult(queryId); @@ -316,9 +316,9 @@ public void run() { progress.set(100); } - } else if (response.getResultCode() == ClientProtos.ResultCode.ERROR) { - if (response.hasErrorMessage()) { - StringBuffer errorMessage = new StringBuffer(response.getErrorMessage()); + } else if (response.getResult().getResultCode() == ClientProtos.ResultCode.ERROR) { + if (response.getResult().hasErrorMessage()) { + StringBuffer errorMessage = new StringBuffer(response.getResult().getErrorMessage()); String modifiedMessage; if (errorMessage.length() > 200) { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java index 1c831107a8..6a13898996 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerClientService.java @@ -25,10 +25,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.StringUtils; import org.apache.tajo.QueryId; +import org.apache.tajo.annotation.Nullable; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.ipc.ClientProtos.GetQueryHistoryResponse; import org.apache.tajo.ipc.ClientProtos.QueryIdRequest; +import org.apache.tajo.ipc.ClientProtos.RequestResult; import org.apache.tajo.ipc.ClientProtos.ResultCode; import org.apache.tajo.ipc.QueryMasterClientProtocol; import org.apache.tajo.master.querymaster.QueryMasterTask; @@ -129,14 +132,32 @@ public GetQueryHistoryResponse getQueryHistory(RpcController controller, QueryId if (queryHistory != null) { builder.setQueryHistory(queryHistory.getProto()); } - builder.setResultCode(ResultCode.OK); + builder.setResult(buildOkRequestResult()); } catch (Throwable t) { LOG.warn(t.getMessage(), t); - builder.setResultCode(ResultCode.ERROR); - builder.setErrorMessage(org.apache.hadoop.util.StringUtils.stringifyException(t)); + builder.setResult(buildRequestResult(ResultCode.ERROR, + StringUtils.stringifyException(t), null)); } return builder.build(); } + + private RequestResult buildOkRequestResult() { + return buildRequestResult(ResultCode.OK, null, null); + } + + private RequestResult buildRequestResult(ResultCode code, + @Nullable String errorMessage, + @Nullable String errorTrace) { + RequestResult.Builder builder = RequestResult.newBuilder(); + builder.setResultCode(code); + if (errorMessage != null) { + builder.setErrorMessage(errorMessage); + } + if (errorTrace != null) { + builder.setErrorTrace(errorTrace); + } + return builder.build(); + } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index 5f9c6ace1c..b784c648e6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -125,21 +125,9 @@ public Task(String taskRunnerId, this.inputStats = new TableStats(); plan = LogicalNodeDeserializer.deserialize(queryContext, request.getPlan()); - LogicalNode [] scanNode = PlannerUtil.findAllNodes(plan, NodeType.SCAN); - if (scanNode != null) { - for (LogicalNode node : scanNode) { - ScanNode scan = (ScanNode) node; - descs.put(scan.getCanonicalName(), scan.getTableDesc()); - } - } - - LogicalNode [] partitionScanNode = PlannerUtil.findAllNodes(plan, NodeType.PARTITIONS_SCAN); - if (partitionScanNode != null) { - for (LogicalNode node : partitionScanNode) { - PartitionedTableScanNode scan = (PartitionedTableScanNode) node; - descs.put(scan.getCanonicalName(), scan.getTableDesc()); - } - } + updateDescsForScanNodes(NodeType.SCAN); + updateDescsForScanNodes(NodeType.PARTITIONS_SCAN); + updateDescsForScanNodes(NodeType.INDEX_SCAN); interQuery = request.getProto().getInterQuery(); if (interQuery) { @@ -181,6 +169,17 @@ public Task(String taskRunnerId, LOG.info("=================================="); } + private void updateDescsForScanNodes(NodeType nodeType) { + assert nodeType == NodeType.SCAN || nodeType == NodeType.PARTITIONS_SCAN || nodeType == NodeType.INDEX_SCAN; + LogicalNode[] scanNodes = PlannerUtil.findAllNodes(plan, nodeType); + if (scanNodes != null) { + for (LogicalNode node : scanNodes) { + ScanNode scanNode = (ScanNode) node; + descs.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); + } + } + } + public void init() throws IOException { if (context.getState() == TaskAttemptState.TA_PENDING) { // initialize a task temporal dir diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index dfc8a9b482..6c0af89a8b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -379,12 +379,15 @@ public Collection getInputTables() { return fragmentMap.get(id).toArray(new FragmentProto[fragmentMap.get(id).size()]); } - public long getUniqueKeyFromFragments() { - List totalFragments = new ArrayList(); - for (List eachFragments : fragmentMap.values()) { - totalFragments.addAll(eachFragments); + public String getUniqueKeyFromFragments() { + StringBuilder sb = new StringBuilder(); + for (List fragments : fragmentMap.values()) { + for (FragmentProto f : fragments) { + FileFragment fileFragment = FragmentConvertor.convert(FileFragment.class, f); + sb.append(fileFragment.getPath().getName()).append(fileFragment.getStartKey()).append(fileFragment.getLength()); + } } - return Objects.hashCode(totalFragments.toArray(new FragmentProto[totalFragments.size()])); + return sb.toString(); } public int hashCode() { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 4e4b710f28..615dbb9681 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -36,11 +36,11 @@ import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.plan.*; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.serder.EvalNodeDeserializer; import org.apache.tajo.plan.serder.EvalNodeSerializer; -import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.plan.verifier.LogicalPlanVerifier; @@ -98,7 +98,7 @@ public static void setUp() throws Exception { analyzer = new SQLAnalyzer(); preLogicalPlanVerifier = new PreLogicalPlanVerifier(cat); planner = new LogicalPlanner(cat); - optimizer = new LogicalOptimizer(util.getConfiguration()); + optimizer = new LogicalOptimizer(util.getConfiguration(), cat); annotatedPlanVerifier = new LogicalPlanVerifier(util.getConfiguration(), cat); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java index c9b52fd93b..3122b25a51 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java @@ -104,9 +104,9 @@ public static void setUp() throws Exception { catalog.createFunction(funcDesc); sqlAnalyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(util.getConfiguration()); defaultContext = LocalTajoTestingUtility.createDummyContext(util.getConfiguration()); + optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); } @AfterClass diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 996d7366fc..0bf4602cb4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -497,7 +497,7 @@ public final void testJoinWithMultipleJoinQual1() throws IOException, PlanningEx Schema expected = tpch.getOutSchema("q2"); assertSchema(expected, node.getOutSchema()); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration()); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); optimizer.optimize(plan); LogicalNode[] nodes = PlannerUtil.findAllNodes(node, NodeType.JOIN); @@ -536,7 +536,7 @@ public final void testJoinWithMultipleJoinQual2() throws IOException, PlanningEx LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration()); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); optimizer.optimize(plan); LogicalNode[] nodes = PlannerUtil.findAllNodes(node, NodeType.SCAN); @@ -577,7 +577,7 @@ public final void testJoinWithMultipleJoinQual3() throws IOException, PlanningEx LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration()); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); optimizer.optimize(plan); LogicalNode[] nodes = PlannerUtil.findAllNodes(node, NodeType.SCAN); @@ -624,7 +624,7 @@ public final void testJoinWithMultipleJoinQual4() throws IOException, PlanningEx LogicalNode node = plan.getRootBlock().getRoot(); testJsonSerDerObject(node); - LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration()); + LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); optimizer.optimize(plan); Map scanMap = TUtil.newHashMap(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index 3803c7a3c6..9f2077669a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -180,7 +180,7 @@ public final void testBroadcastJoin() throws IOException, PlanningException { "join small2 on small1_id = small2_id"; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -241,7 +241,7 @@ public final void testBroadcastJoinAllSmallTables() throws IOException, Planning "join small3 on small1_id = small3_id"; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -305,7 +305,7 @@ public final void testNotBroadcastJoinTwoLargeTable() throws IOException, Planni "join large2 on large1_id = large2_id "; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -333,7 +333,7 @@ public final void testTwoBroadcastJoin() throws IOException, PlanningException { "join small2 on large2_id = small2_id"; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -383,7 +383,7 @@ public final void testNotBroadcastJoinSubquery() throws IOException, PlanningExc "join small2 on a.small1_id = small2_id"; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -424,7 +424,7 @@ public final void testBroadcastJoinSubquery() throws IOException, PlanningExcept "join (select * from small1) a on large1_id = a.small1_id"; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -480,7 +480,7 @@ public final void testLeftOuterJoinCase1() throws IOException, PlanningException "left outer join large2 on small1_id = large2_id "; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -534,7 +534,7 @@ public final void testLeftOuterJoinCase2() throws IOException, PlanningException "left outer join small3 on large1_id = small3_id "; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -617,7 +617,7 @@ public final void testLeftOuterJoinCase3() throws IOException, PlanningException "left outer join small3 on large3_id = small3_id "; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -700,7 +700,7 @@ public final void testLeftOuterJoinCase4() throws IOException, PlanningException "left outer join small3 on small1_id = small3_id "; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -759,7 +759,7 @@ public final void testLeftOuterJoinCase5() throws IOException, PlanningException "left outer join small3 on small1_id = small3_id " ; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -812,7 +812,7 @@ public final void testLeftOuterJoinCase6() throws IOException, PlanningException "left outer join small3 on small1_id = small3_id " ; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -904,7 +904,7 @@ public final void testInnerLeftOuterJoinCase1() throws IOException, PlanningExce "left outer join small3 on small3_id = large1_id " ; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); @@ -969,7 +969,7 @@ public final void testBroadcastCasebyCase1() throws IOException, PlanningExcepti "left outer join small2 on large1_id = small2_id " ; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java deleted file mode 100644 index c897461c95..0000000000 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.planner.physical; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.LocalTajoTestingUtility; -import org.apache.tajo.TajoConstants; -import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.algebra.Expr; -import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.plan.LogicalOptimizer; -import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.plan.LogicalPlanner; -import org.apache.tajo.engine.planner.PhysicalPlannerImpl; -import org.apache.tajo.plan.logical.LogicalNode; -import org.apache.tajo.plan.logical.ScanNode; -import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.storage.fragment.FragmentConvertor; -import org.apache.tajo.storage.index.bst.BSTIndex; -import org.apache.tajo.util.CommonTestingUtil; -import org.apache.tajo.worker.TaskAttemptContext; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Random; -import java.util.Stack; - -import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; -import static org.junit.Assert.assertEquals; - -public class TestBSTIndexExec { - - private TajoConf conf; - private Path idxPath; - private CatalogService catalog; - private SQLAnalyzer analyzer; - private LogicalPlanner planner; - private LogicalOptimizer optimizer; - private FileStorageManager sm; - private Schema idxSchema; - private BaseTupleComparator comp; - private BSTIndex.BSTIndexWriter writer; - private HashMap randomValues ; - private int rndKey = -1; - private FileSystem fs; - private TableMeta meta; - private Path tablePath; - - private Random rnd = new Random(System.currentTimeMillis()); - - private TajoTestingCluster util; - - @Before - public void setup() throws Exception { - this.randomValues = new HashMap(); - this.conf = new TajoConf(); - util = new TajoTestingCluster(); - util.startCatalogCluster(); - catalog = util.getMiniCatalogCluster().getCatalog(); - - Path workDir = CommonTestingUtil.getTestDir(); - catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); - catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = (FileStorageManager)StorageManager.getFileStorageManager(conf, workDir); - - idxPath = new Path(workDir, "test.idx"); - - Schema schema = new Schema(); - schema.addColumn("managerid", Type.INT4); - schema.addColumn("empid", Type.INT4); - schema.addColumn("deptname", Type.TEXT); - - this.idxSchema = new Schema(); - idxSchema.addColumn("managerid", Type.INT4); - SortSpec[] sortKeys = new SortSpec[1]; - sortKeys[0] = new SortSpec(idxSchema.getColumn("managerid"), true, false); - this.comp = new BaseTupleComparator(idxSchema, sortKeys); - - this.writer = new BSTIndex(conf).getIndexWriter(idxPath, - BSTIndex.TWO_LEVEL_INDEX, this.idxSchema, this.comp); - writer.setLoadNum(100); - writer.open(); - long offset; - - meta = CatalogUtil.newTableMeta(StoreType.CSV); - tablePath = StorageUtil.concatPath(workDir, "employee", "table.csv"); - fs = tablePath.getFileSystem(conf); - fs.mkdirs(tablePath.getParent()); - - FileAppender appender = (FileAppender)sm.getAppender(meta, schema, tablePath); - appender.init(); - Tuple tuple = new VTuple(schema.size()); - for (int i = 0; i < 10000; i++) { - - Tuple key = new VTuple(this.idxSchema.size()); - int rndKey = rnd.nextInt(250); - if(this.randomValues.containsKey(rndKey)) { - int t = this.randomValues.remove(rndKey) + 1; - this.randomValues.put(rndKey, t); - } else { - this.randomValues.put(rndKey, 1); - } - - key.put(new Datum[] { DatumFactory.createInt4(rndKey) }); - tuple.put(new Datum[] { DatumFactory.createInt4(rndKey), - DatumFactory.createInt4(rnd.nextInt(10)), - DatumFactory.createText("dept_" + rnd.nextInt(10)) }); - offset = appender.getOffset(); - appender.addTuple(tuple); - writer.write(key, offset); - } - appender.flush(); - appender.close(); - writer.close(); - - TableDesc desc = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, - sm.getTablePath("employee").toUri()); - catalog.createTable(desc); - - analyzer = new SQLAnalyzer(); - planner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(conf); - } - - @After - public void tearDown() { - util.shutdownCatalogCluster(); - } - - @Test - public void testEqual() throws Exception { - this.rndKey = rnd.nextInt(250); - final String QUERY = "select * from employee where managerId = " + rndKey; - - FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE); - Path workDir = CommonTestingUtil.getTestDir("target/test-data/testEqual"); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), - LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir); - Expr expr = analyzer.parse(QUERY); - LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); - LogicalNode rootNode = optimizer.optimize(plan); - - TmpPlanner phyPlanner = new TmpPlanner(conf); - PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); - - int tupleCount = this.randomValues.get(rndKey); - int counter = 0; - exec.init(); - while (exec.next() != null) { - counter ++; - } - exec.close(); - assertEquals(tupleCount , counter); - } - - private class TmpPlanner extends PhysicalPlannerImpl { - public TmpPlanner(TajoConf conf) { - super(conf); - } - - @Override - public PhysicalExec createScanPlan(TaskAttemptContext ctx, ScanNode scanNode, Stack stack) - throws IOException { - Preconditions.checkNotNull(ctx.getTable(scanNode.getTableName()), - "Error: There is no table matched to %s", scanNode.getTableName()); - - List fragments = FragmentConvertor.convert(ctx.getConf(), ctx.getTables(scanNode.getTableName())); - - Datum[] datum = new Datum[]{DatumFactory.createInt4(rndKey)}; - - return new BSTIndexScanExec(ctx, scanNode, fragments.get(0), idxPath, idxSchema, comp , datum); - - } - } -} \ No newline at end of file diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 64da88bda7..a64b525793 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -63,6 +63,7 @@ public class TestHashAntiJoinExec { private LogicalOptimizer optimizer; private StorageManager sm; private Path testDir; + private QueryContext queryContext; private TableDesc employee; private TableDesc people; @@ -128,11 +129,12 @@ public void setUp() throws Exception { appender.flush(); appender.close(); + queryContext = new QueryContext(conf); people = CatalogUtil.newTableDesc("default.people", peopleSchema, peopleMeta, peoplePath); catalog.createTable(people); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(conf); + optimizer = new LogicalOptimizer(conf, catalog); } @After @@ -159,7 +161,7 @@ public final void testHashAntiJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashAntiJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(queryContext, LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 4e218c537f..196f3bf442 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -64,6 +64,7 @@ public class TestHashSemiJoinExec { private LogicalOptimizer optimizer; private StorageManager sm; private Path testDir; + private QueryContext queryContext; private TableDesc employee; private TableDesc people; @@ -133,11 +134,12 @@ public void setUp() throws Exception { appender.flush(); appender.close(); + queryContext = new QueryContext(conf); people = CatalogUtil.newTableDesc("default.people", peopleSchema, peopleMeta, peoplePath); catalog.createTable(people); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(conf); + optimizer = new LogicalOptimizer(conf, catalog); } @After @@ -164,7 +166,7 @@ public final void testHashSemiJoin() throws IOException, PlanningException { FileFragment[] merged = TUtil.concat(empFrags, peopleFrags); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashSemiJoin"); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(queryContext, LocalTajoTestingUtility.newTaskAttemptId(), merged, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index a4e49f7c8c..9de58a3fbe 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -171,12 +171,13 @@ public static void setUp() throws Exception { } appender.flush(); appender.close(); + + defaultContext = LocalTajoTestingUtility.createDummyContext(conf); catalog.createTable(score); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(conf); + optimizer = new LogicalOptimizer(conf, catalog); - defaultContext = LocalTajoTestingUtility.createDummyContext(conf); masterPlan = new MasterPlan(LocalTajoTestingUtility.newQueryId(), null, null); createLargeScoreTable(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 2e093c1efd..84abfffc6e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -61,6 +61,7 @@ public class TestSortExec { private static Path workDir; private static Path tablePath; private static TableMeta employeeMeta; + private static QueryContext queryContext; private static Random rnd = new Random(System.currentTimeMillis()); @@ -101,9 +102,10 @@ public static void setUp() throws Exception { tablePath.toUri()); catalog.createTable(desc); + queryContext = new QueryContext(conf); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(conf); + optimizer = new LogicalOptimizer(conf, catalog); } public static String[] QUERIES = { @@ -113,7 +115,7 @@ public static void setUp() throws Exception { public final void testNext() throws IOException, PlanningException { FileFragment[] frags = FileStorageManager.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestSortExec"); - TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), + TaskAttemptContext ctx = new TaskAttemptContext(queryContext, LocalTajoTestingUtility .newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java index 2a66909640..83e33b7ef2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java @@ -23,6 +23,7 @@ import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.IndexDesc; import org.apache.tajo.conf.TajoConf.ConfVars; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -38,35 +39,60 @@ public TestCreateIndex() { super(TajoConstants.DEFAULT_DATABASE_NAME); } - private static void assertIndexExist(String indexName) throws IOException { - Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName); + private static void assertIndexNotExist(String databaseName, String indexName) throws IOException { + Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), databaseName + "/" + indexName); FileSystem fs = indexPath.getFileSystem(conf); - assertTrue(fs.exists(indexPath)); - assertEquals(2, fs.listStatus(indexPath).length); - fs.deleteOnExit(indexPath); + if (fs.exists(indexPath)) { + fs.deleteOnExit(indexPath); + assertFalse("Index is not deleted from the file system.", true); + } } @Test public final void testCreateIndex() throws Exception { executeQuery(); - assertIndexExist("l_orderkey_idx"); + assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); + assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey"})); + executeString("drop index l_orderkey_idx"); + assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); + assertIndexNotExist(getCurrentDatabase(), "l_orderkey_idx"); } @Test public final void testCreateIndexOnMultiAttrs() throws Exception { executeQuery(); - assertIndexExist("l_orderkey_partkey_idx"); + assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_partkey_idx")); + assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey", "l_partkey"})); + executeString("drop index l_orderkey_partkey_idx"); + assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_partkey_idx")); + assertIndexNotExist(getCurrentDatabase(), "l_orderkey_partkey_idx"); } @Test public final void testCreateIndexWithCondition() throws Exception { executeQuery(); - assertIndexExist("l_orderkey_partkey_lt10_idx"); + assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_partkey_lt10_idx")); + assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey", "l_partkey"})); + executeString("drop index l_orderkey_partkey_lt10_idx"); + assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_partkey_lt10_idx")); + assertIndexNotExist(getCurrentDatabase(), "l_orderkey_partkey_lt10_idx"); } @Test public final void testCreateIndexOnExpression() throws Exception { executeQuery(); - assertIndexExist("l_orderkey_100_lt10_idx"); + assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_lt10_idx")); + executeString("drop index l_orderkey_100_lt10_idx"); + assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_lt10_idx")); + assertIndexNotExist(getCurrentDatabase(), "l_orderkey_100_lt10_idx"); + } + + @Test + public final void testCreateIndexOnMultiExprs() throws Exception { + executeQuery(); + assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx")); + executeString("drop index l_orderkey_100_l_linenumber_10_lt10_idx"); + assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx")); + assertIndexNotExist(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx"); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java new file mode 100644 index 0000000000..f94a3b545e --- /dev/null +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import com.google.protobuf.ServiceException; +import org.apache.tajo.IntegrationTest; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; + +@Category(IntegrationTest.class) +public class TestIndexScan extends QueryTestCaseBase { + + public TestIndexScan() throws ServiceException { + super(TajoConstants.DEFAULT_DATABASE_NAME); + Map sessionVars = new HashMap(); + sessionVars.put(SessionVars.INDEX_ENABLED.keyname(), "true"); + sessionVars.put(SessionVars.INDEX_SELECTIVITY_THRESHOLD.keyname(), "0.01f"); + client.updateSessionVariables(sessionVars); + } + + @Test + public final void testOnSortedNonUniqueKeys() throws Exception { + executeString("create index l_orderkey_idx on lineitem (l_orderkey)"); + ResultSet res = executeString("select * from lineitem where l_orderkey = 1;"); + assertResultSet(res); + cleanupQuery(res); + executeString("drop index l_orderkey_idx"); + } + + @Test + public final void testOnUnsortedTextKeys() throws Exception { + executeString("create index l_shipdate_idx on lineitem (l_shipdate)"); + ResultSet res = executeString("select l_orderkey, l_shipdate, l_comment from lineitem where l_shipdate = '1997-01-28';"); + assertResultSet(res); + cleanupQuery(res); + executeString("drop index l_shipdate_idx"); + } + + @Test + public final void testOnMultipleKeys() throws Exception { + executeString("create index multikey_idx on lineitem (l_shipdate asc null last, l_tax desc null first, l_shipmode, l_linenumber desc null last)"); + ResultSet res = executeString("select l_orderkey, l_shipdate, l_comment from lineitem " + + "where l_shipdate = '1997-01-28' and l_tax = 0.05 and l_shipmode = 'RAIL' and l_linenumber = 1;"); + assertResultSet(res); + cleanupQuery(res); + executeString("drop index multikey_idx"); + } + + @Test + public final void testOnMultipleKeys2() throws Exception { + executeString("create index multikey_idx on lineitem (l_shipdate asc null last, l_tax desc null first)"); + ResultSet res = executeString("select l_orderkey, l_shipdate, l_comment from lineitem " + + "where l_shipdate = '1997-01-28' and l_tax = 0.05 and l_shipmode = 'RAIL' and l_linenumber = 1;"); + assertResultSet(res); + cleanupQuery(res); + executeString("drop index multikey_idx"); + } + + @Test + public final void testOnMultipleExprs() throws Exception { + executeString("create index l_orderkey_100_l_linenumber_10_idx on lineitem (l_orderkey*100-l_linenumber*10 asc null first);"); + ResultSet res = executeString("select l_orderkey, l_linenumber from lineitem where l_orderkey*100-l_linenumber*10 = 280"); + assertResultSet(res); + cleanupQuery(res); + executeString("drop index l_orderkey_100_l_linenumber_10_idx"); + } + + @Test + public final void testWithGroupBy() throws Exception { + executeString("create index l_shipdate_idx on lineitem (l_shipdate)"); + ResultSet res = executeString("select l_shipdate, count(*) from lineitem where l_shipdate = '1997-01-28' group by l_shipdate;"); + assertResultSet(res); + cleanupQuery(res); + executeString("drop index l_shipdate_idx"); + } + + @Test + public final void testWithSort() throws Exception { + executeString("create index l_orderkey_idx on lineitem (l_orderkey)"); + ResultSet res = executeString("select l_shipdate from lineitem where l_orderkey = 1 order by l_shipdate;"); + assertResultSet(res); + cleanupQuery(res); + executeString("drop index l_orderkey_idx"); + } + + @Test + public final void testWithJoin() throws Exception { + executeString("create index l_orderkey_idx on lineitem (l_orderkey)"); + executeString("create index o_orderkey_idx on orders (o_orderkey)"); + ResultSet res = executeString("select l_shipdate, o_orderstatus from lineitem, orders where l_orderkey = o_orderkey and l_orderkey = 1 and o_orderkey = 1;"); + assertResultSet(res); + cleanupQuery(res); + executeString("drop index l_orderkey_idx"); + executeString("drop index o_orderkey_idx"); + } +} diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index 3400752fbb..0bfcb09a6a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -766,8 +766,8 @@ public final void testColumnPartitionedTableWithSmallerExpressions1() throws Exc ClientProtos.SubmitQueryResponse response = client.executeQuery("insert overwrite into " + tableName + " select l_orderkey, l_partkey from lineitem"); - assertTrue(response.hasErrorMessage()); - assertEquals(response.getErrorMessage(), "INSERT has smaller expressions than target columns\n"); + assertTrue(response.getResult().hasErrorMessage()); + assertEquals(response.getResult().getErrorMessage(), "INSERT has smaller expressions than target columns\n"); res = executeFile("case14.sql"); assertResultSet(res, "case14.result"); @@ -786,8 +786,8 @@ public final void testColumnPartitionedTableWithSmallerExpressions2() throws Exc ClientProtos.SubmitQueryResponse response = client.executeQuery("insert overwrite into " + tableName + " select l_returnflag , l_orderkey, l_partkey from lineitem"); - assertTrue(response.hasErrorMessage()); - assertEquals(response.getErrorMessage(), "INSERT has smaller expressions than target columns\n"); + assertTrue(response.getResult().hasErrorMessage()); + assertEquals(response.getResult().getErrorMessage(), "INSERT has smaller expressions than target columns\n"); res = executeFile("case15.sql"); assertResultSet(res, "case15.result"); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 712243bf61..760cb4cb40 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -80,7 +80,7 @@ public static void setUp() throws Exception { analyzer = new SQLAnalyzer(); logicalPlanner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(conf); + optimizer = new LogicalOptimizer(conf, catalog); StorageManager sm = StorageManager.getFileStorageManager(conf); dispatcher = new AsyncDispatcher(); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java index d0f7cf44a8..e2f341759c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java @@ -115,7 +115,7 @@ public static void setUp() throws Exception { sqlAnalyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(util.getConfiguration()); + optimizer = new LogicalOptimizer(util.getConfiguration(), catalog); globalPlanner = new GlobalPlanner(util.getConfiguration(), catalog); } diff --git a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java index 8ca4cfff49..e728207818 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java @@ -76,7 +76,7 @@ public final void testKillQueryFromInitState() throws Exception { String query = "select l_orderkey, l_partkey from lineitem group by l_orderkey, l_partkey order by l_orderkey"; LogicalPlanner planner = new LogicalPlanner(catalog); - LogicalOptimizer optimizer = new LogicalOptimizer(conf); + LogicalOptimizer optimizer = new LogicalOptimizer(conf, catalog); Expr expr = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, expr); diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index 200ba317e6..2eecd4df75 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -94,7 +94,7 @@ public void setUp() throws Exception { analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); - optimizer = new LogicalOptimizer(conf); + optimizer = new LogicalOptimizer(conf, catalog); schema = new Schema(); schema.addColumn("empid", Type.INT4); diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiExprs.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiExprs.sql new file mode 100644 index 0000000000..7938005da0 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnMultiExprs.sql @@ -0,0 +1 @@ +create index l_orderkey_100_l_linenumber_10_lt10_idx on lineitem (l_orderkey*100-l_linenumber*10 asc null first) where l_orderkey*100 > 10; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleExprs.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleExprs.result new file mode 100644 index 0000000000..bcb645d28a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleExprs.result @@ -0,0 +1,3 @@ +l_orderkey,l_linenumber +------------------------------- +3,2 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys.result new file mode 100644 index 0000000000..86d468efc6 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys.result @@ -0,0 +1,3 @@ +l_orderkey,l_shipdate,l_comment +------------------------------- +2,1997-01-28,ven requests. deposits breach a \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys2.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys2.result new file mode 100644 index 0000000000..86d468efc6 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnMultipleKeys2.result @@ -0,0 +1,3 @@ +l_orderkey,l_shipdate,l_comment +------------------------------- +2,1997-01-28,ven requests. deposits breach a \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnSortedNonUniqueKeys.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnSortedNonUniqueKeys.result new file mode 100644 index 0000000000..fb8a4c248b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnSortedNonUniqueKeys.result @@ -0,0 +1,4 @@ +l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment +------------------------------- +1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the +1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testOnUnsortedTextKeys.result b/tajo-core/src/test/resources/results/TestIndexScan/testOnUnsortedTextKeys.result new file mode 100644 index 0000000000..86d468efc6 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestIndexScan/testOnUnsortedTextKeys.result @@ -0,0 +1,3 @@ +l_orderkey,l_shipdate,l_comment +------------------------------- +2,1997-01-28,ven requests. deposits breach a \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testWithGroupBy.result b/tajo-core/src/test/resources/results/TestIndexScan/testWithGroupBy.result new file mode 100644 index 0000000000..2d1d12eb17 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestIndexScan/testWithGroupBy.result @@ -0,0 +1,3 @@ +l_shipdate,?count +------------------------------- +1997-01-28,1 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testWithJoin.result b/tajo-core/src/test/resources/results/TestIndexScan/testWithJoin.result new file mode 100644 index 0000000000..d119969587 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestIndexScan/testWithJoin.result @@ -0,0 +1,4 @@ +l_shipdate,o_orderstatus +------------------------------- +1996-03-13,O +1996-04-12,O \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestIndexScan/testWithSort.result b/tajo-core/src/test/resources/results/TestIndexScan/testWithSort.result new file mode 100644 index 0000000000..774a4117c1 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestIndexScan/testWithSort.result @@ -0,0 +1,4 @@ +l_shipdate +------------------------------- +1996-03-13 +1996-04-12 \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result index b5b7c229c9..90f948edc0 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result @@ -34,6 +34,8 @@ Available Session Variables: \set MAX_OUTPUT_FILE_SIZE [int value] - Maximum per-output file size (mb). 0 means infinite. \set NULL_CHAR [text value] - null char of text file output \set CODEGEN [true or false] - Runtime code generation enabled (experiment) +\set INDEX_ENABLED [true or false] - index scan enabled +\set INDEX_SELECTIVITY_THRESHOLD [real value] - the selectivity threshold for index scan \set ARITHABORT [true or false] - If true, a running query will be terminated when an overflow or divide-by-zero occurs. \set FETCH_ROWNUM [int value] - Sets the number of rows at a time from Master \set DEBUG_ENABLED [true or false] - (debug only) debug mode enabled \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/index.rst b/tajo-docs/src/main/sphinx/index.rst index 80cd8428e9..667f2700ca 100644 --- a/tajo-docs/src/main/sphinx/index.rst +++ b/tajo-docs/src/main/sphinx/index.rst @@ -37,6 +37,7 @@ Table of Contents: functions table_management table_partitioning + index_overview backup_and_restore hcatalog_integration jdbc_driver diff --git a/tajo-docs/src/main/sphinx/index/future_work.rst b/tajo-docs/src/main/sphinx/index/future_work.rst new file mode 100644 index 0000000000..c6ec47d6a7 --- /dev/null +++ b/tajo-docs/src/main/sphinx/index/future_work.rst @@ -0,0 +1,8 @@ +************************************* +Future Works +************************************* + +* Providing more index types, such as bitmap and HBase index +* Supporting index on partitioned tables +* Supporting the backup and restore feature +* Cost-based query optimization by estimating the query selectivity \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/index/how_to_use.rst b/tajo-docs/src/main/sphinx/index/how_to_use.rst new file mode 100644 index 0000000000..82a8bc9534 --- /dev/null +++ b/tajo-docs/src/main/sphinx/index/how_to_use.rst @@ -0,0 +1,69 @@ +************************************* +How to use index? +************************************* + +------------------------------------- +1. Create index +------------------------------------- + +The first step for utilizing index is index creation. You can create index using SQL (:doc:`/sql_language/ddl`) or Tajo API (:doc:`/tajo_client_api`). For example, you can create a BST index on the lineitem table by submitting the following SQL to Tajo. + +.. code-block:: sql + + create index l_orderkey_idx on lineitem (l_orderkey); + +If the index is created successfully, you can see the information about that index as follows: :: + + default> \d lineitem + + table name: default.lineitem + table path: hdfs://localhost:7020/tpch/lineitem + store type: CSV + number of rows: unknown + volume: 753.9 MB + Options: + 'text.delimiter'='|' + + schema: + l_orderkey INT8 + l_partkey INT8 + l_suppkey INT8 + l_linenumber INT8 + l_quantity FLOAT4 + l_extendedprice FLOAT4 + l_discount FLOAT4 + l_tax FLOAT4 + l_returnflag TEXT + l_linestatus TEXT + l_shipdate DATE + l_commitdate DATE + l_receiptdate DATE + l_shipinstruct TEXT + l_shipmode TEXT + l_comment TEXT + + + Indexes: + "l_orderkey_idx" TWO_LEVEL_BIN_TREE (l_orderkey ASC NULLS LAST ) + +For more information about index creation, please refer to the above links. + +------------------------------------- +2. Enable/disable index scans +------------------------------------- + +When an index is successfully created, you must enable the index scan feature as follows: + +.. code-block:: sql + + \set INDEX_ENABLED true + +If you don't want to use the index scan feature anymore, you can simply disable it as follows: + +.. code-block:: sql + + \set INDEX_ENABLED false + +.. note:: + + If the index scan feature is enabled, Tajo currently always performs index scan regardless of its efficiency. You should set this option when the expected number of retrieved tuples is sufficiently small. \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/index/types.rst b/tajo-docs/src/main/sphinx/index/types.rst new file mode 100644 index 0000000000..457f45351d --- /dev/null +++ b/tajo-docs/src/main/sphinx/index/types.rst @@ -0,0 +1,7 @@ +************************************* +Index Types +************************************* + +Currently, Tajo supports only one type of index, ``TWO_LEVEL_BIN_TREE``, shortly ``BST``. The BST index is a kind of binary search tree which is extended to be permanently stored on disk. It consists of two levels of nodes; a leaf node indexes the keys with the positions of data in an HDFS block and a root node indexes the keys with the leaf node indices. + +When an index scan is started, the query engine first reads the root node and finds the search key. If it finds a leaf node corresponding to the search key, it subsequently finds the search key in that leaf node. Finally, it directly reads a tuple corresponding to the search key from HDFS. \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/index_overview.rst b/tajo-docs/src/main/sphinx/index_overview.rst new file mode 100644 index 0000000000..78ecb497ce --- /dev/null +++ b/tajo-docs/src/main/sphinx/index_overview.rst @@ -0,0 +1,20 @@ +*********************** +Index (Experimental Feature) +*********************** + +An index is a data structure that is used for efficient query processing. Using an index, the Tajo query engine can directly retrieve search values. + +This is still an experimental feature. In order to use indexes, you must check out the source code of the ``index_support`` branch:: + + git clone -b index_support https://git-wip-us.apache.org/repos/asf/tajo.git tajo-index + +For the source code build, please refer to :doc:`getting_started`. + +The following sections describe the supported index types, the query execution with an index, and the future works. + +.. toctree:: + :maxdepth: 1 + + index/types + index/how_to_use + index/future_work \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/sql_language/ddl.rst b/tajo-docs/src/main/sphinx/sql_language/ddl.rst index 3fba6beed1..60b7190c20 100644 --- a/tajo-docs/src/main/sphinx/sql_language/ddl.rst +++ b/tajo-docs/src/main/sphinx/sql_language/ddl.rst @@ -75,4 +75,35 @@ If you want to add an external table that contains compressed data, you should g DROP TABLE [IF EXISTS] [PURGE] -``IF EXISTS`` allows ``DROP DATABASE`` statement to avoid an error which occurs when the database does not exist. ``DROP TABLE`` statement removes a table from Tajo catalog, but it does not remove the contents. If ``PURGE`` option is given, ``DROP TABLE`` statement will eliminate the entry in the catalog as well as the contents. \ No newline at end of file +``IF EXISTS`` allows ``DROP DATABASE`` statement to avoid an error which occurs when the database does not exist. ``DROP TABLE`` statement removes a table from Tajo catalog, but it does not remove the contents. If ``PURGE`` option is given, ``DROP TABLE`` statement will eliminate the entry in the catalog as well as the contents. + +======================== + CREATE INDEX +======================== + +*Synopsis* + +.. code-block:: sql + + CREATE INDEX [ name ] ON table_name [ USING method ] + ( { column_name | ( expression ) } [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [, ...] ) + [ WHERE predicate ] + +------------------------ + Index method +------------------------ + +Currently, Tajo supports only one type of index. + +Index methods: + * TWO_LEVEL_BIN_TREE: This method is used by default in Tajo. For more information about its structure, please refer to :doc:`/index/types`. + +======================== + DROP INDEX +======================== + +*Synopsis* + +.. code-block:: sql + + DROP INDEX name \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 18a8859d19..ed16f82057 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -28,6 +28,7 @@ import org.apache.tajo.OverridableConf; import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; +import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.util.ReflectionUtil; @@ -39,6 +40,10 @@ import org.apache.tajo.plan.joinorder.JoinGraph; import org.apache.tajo.plan.joinorder.JoinOrderAlgorithm; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.rules.AccessPathRewriter; +import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule; +import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; +import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; import org.apache.tajo.plan.rewrite.*; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; @@ -57,12 +62,15 @@ public class LogicalOptimizer { private static final Log LOG = LogFactory.getLog(LogicalOptimizer.class.getName()); + private CatalogService catalog; private BaseLogicalPlanRewriteEngine rulesBeforeJoinOpt; private BaseLogicalPlanRewriteEngine rulesAfterToJoinOpt; private JoinOrderAlgorithm joinOrderAlgorithm = new GreedyHeuristicJoinOrderAlgorithm(); - public LogicalOptimizer(TajoConf conf) { + public LogicalOptimizer(TajoConf conf, CatalogService catalog) { + this.catalog = catalog; + // TODO: set the catalog instance to FilterPushdownRule Class clazz = conf.getClassVar(ConfVars.LOGICAL_PLAN_REWRITE_RULE_PROVIDER_CLASS); LogicalPlanRewriteRuleProvider provider = (LogicalPlanRewriteRuleProvider) ReflectionUtil.newInstance(clazz, conf); @@ -86,7 +94,7 @@ public LogicalNode optimize(LogicalPlan plan) throws PlanningException { } public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws PlanningException { - rulesBeforeJoinOpt.rewrite(context, plan); + rulesBeforeJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog)); DirectedGraphCursor blockCursor = new DirectedGraphCursor(plan.getQueryBlockGraph(), plan.getRootBlock().getName()); @@ -99,7 +107,7 @@ public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws Pl } else { LOG.info("Skip Join Optimized."); } - rulesAfterToJoinOpt.rewrite(context, plan); + rulesAfterToJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog)); return plan.getRootBlock().getRoot(); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index 3baf61db81..1535882040 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -25,6 +25,8 @@ import org.apache.tajo.annotation.NotThreadSafe; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; +import org.apache.tajo.plan.expr.AlgebraicUtil.IdentifiableNameBuilder; +import org.apache.tajo.plan.rewrite.rules.AccessPathInfo; import org.apache.tajo.util.graph.DirectedGraphCursor; import org.apache.tajo.util.graph.SimpleDirectedGraph; import org.apache.tajo.plan.expr.ConstEval; @@ -150,6 +152,8 @@ public String generateUniqueColumnName(EvalNode evalNode) { /** * It generates an unique column name from Expr. It is usually used for an expression or predicate without * a specified name (i.e., alias). + * Here, some expressions require to be identified with their names in the future. + * For example, expressions must be identifiable with their names when getting targets in {@link LogicalPlanner#visitCreateIndex}. */ public String generateUniqueColumnName(Expr expr) { String generatedName; @@ -161,6 +165,11 @@ public String generateUniqueColumnName(Expr expr) { return generatedName; } + private String generateUniqueIdentifiableColumnName(Expr expr) { + IdentifiableNameBuilder nameBuilder = new IdentifiableNameBuilder(expr); + return nameBuilder.build(); + } + /** * It attaches a generated column name with a sequence id. It always keeps generated names unique. */ @@ -408,6 +417,7 @@ public class QueryBlock { private final Map columnAliasMap = TUtil.newHashMap(); private final Map> operatorToExprMap = TUtil.newHashMap(); private final List relationList = TUtil.newList(); + private final Map> relNodePidAccessPathMap = TUtil.newHashMap(); private boolean hasWindowFunction = false; private final Map constantPoolByRef = Maps.newHashMap(); private final Map constantPool = Maps.newHashMap(); @@ -496,12 +506,30 @@ public void addRelation(RelationNode relation) { } canonicalNameToRelationMap.put(relation.getCanonicalName(), relation); relationList.add(relation); + relNodePidAccessPathMap.put(relation.getPID(), new ArrayList()); + } + + public void addRelation(RelationNode relation, List accessPathInfos) { + if (relation.hasAlias()) { + TUtil.putToNestedList(relationAliasMap, relation.getTableName(), relation.getCanonicalName()); + } + canonicalNameToRelationMap.put(relation.getCanonicalName(), relation); + relationList.add(relation); + relNodePidAccessPathMap.put(relation.getPID(), new ArrayList()); + } + + public void addAccessPath(RelationNode relation, AccessPathInfo accessPathInfo) { + relNodePidAccessPathMap.get(relation.getPID()).add(accessPathInfo); } public Collection getRelations() { return Collections.unmodifiableList(relationList); } + public List getAccessInfos(RelationNode relation) { + return Collections.unmodifiableList(relNodePidAccessPathMap.get(relation.getPID())); + } + public boolean hasTableExpression() { return this.canonicalNameToRelationMap.size() > 0; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java index 6e2b59a5c0..76a32a2bf5 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java @@ -231,7 +231,8 @@ public LogicalNode visitProjection(LogicalPlanner.PlanContext ctx, Stack s return projectionNode; } - private Target [] buildTargets(LogicalPlanner.PlanContext context, NamedExpr [] exprs) throws PlanningException { + private Target [] buildTargets(LogicalPlanner.PlanContext context, NamedExpr [] exprs) + throws PlanningException { Target [] targets = new Target[exprs.length]; for (int i = 0; i < exprs.length; i++) { NamedExpr namedExpr = exprs[i]; @@ -480,6 +481,11 @@ public LogicalNode visitCreateIndex(LogicalPlanner.PlanContext ctx, Stack return createIndex; } + @Override + public LogicalNode visitDropIndex(LogicalPlanner.PlanContext ctx, Stack stack, DropIndex expr) { + return ctx.plan.createNode(DropIndexNode.class); + } + public LogicalNode visitTruncateTable(LogicalPlanner.PlanContext ctx, Stack stack, TruncateTable expr) throws PlanningException { TruncateTableNode truncateTableNode = ctx.plan.createNode(TruncateTableNode.class); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 3604e06488..b50ef7831d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -41,7 +41,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.plan.LogicalPlan.QueryBlock; import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; @@ -52,12 +51,12 @@ import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; import org.apache.tajo.plan.util.ExprFinder; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; +import java.net.URI; import java.util.*; import static org.apache.tajo.algebra.CreateTable.PartitionType; @@ -1930,9 +1929,9 @@ public LogicalNode visitAlterTable(PlanContext context, Stack stack, Alter return alterTableNode; } - private static Path getIndexPath(PlanContext context, String databaseName, String indexName) { + private static URI getIndexPath(PlanContext context, String databaseName, String indexName) { return new Path(TajoConf.getWarehouseDir(context.queryContext.getConf()), - databaseName + "/" + indexName + "/"); + databaseName + "/" + indexName + "/").toUri(); } @Override @@ -1960,15 +1959,20 @@ public LogicalNode visitCreateIndex(PlanContext context, Stack stack, Crea normalizedExprList[i] = normalizer.normalize(context, sortSpecs[i].getKey()); } for (int i = 0; i < sortKeyNum; i++) { + // even if base expressions don't have their name, + // reference names should be identifiable for the later sort spec creation. referNames[i] = block.namedExprsMgr.addExpr(normalizedExprList[i].baseExpr); block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].aggExprs); block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs); } - createIndexNode.setSortSpecs(annotateSortSpecs(block, referNames, sortSpecs)); - createIndexNode.setIndexType(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase())); - createIndexNode.setIndexPath(getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE), - createIndex.getIndexName())); + Collection relations = block.getRelations(); + assert relations.size() == 1; + createIndexNode.setKeySortSpecs(relations.iterator().next().getLogicalSchema(), + annotateSortSpecs(block, referNames, sortSpecs)); + createIndexNode.setIndexMethod(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase())); + createIndexNode.setIndexPath( + getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName())); if (createIndex.getParams() != null) { KeyValueSet keyValueSet = new KeyValueSet(); @@ -1980,6 +1984,13 @@ public LogicalNode visitCreateIndex(PlanContext context, Stack stack, Crea return createIndexNode; } + @Override + public LogicalNode visitDropIndex(PlanContext context, Stack stack, DropIndex dropIndex) { + DropIndexNode dropIndexNode = context.queryBlock.getNodeFromExpr(dropIndex); + dropIndexNode.setIndexName(dropIndex.getIndexName()); + return dropIndexNode; + } + @Override public LogicalNode visitTruncateTable(PlanContext context, Stack stack, TruncateTable truncateTable) throws PlanningException { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java index 991860ff33..63c10ae8ab 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java @@ -141,6 +141,7 @@ public String getOriginalName(String aliasName) { /** * Adds an expression and returns a reference name. + * @param expr added expression */ public String addExpr(Expr expr) throws PlanningException { if (idToExprBiMap.inverse().containsKey(expr)) { @@ -205,7 +206,8 @@ public String addNamedExpr(NamedExpr namedExpr) throws PlanningException { * Adds a list of expressions and returns a list of reference names. * If some NamedExpr has an alias, NamedExprsManager specifies the alias for the NamedExpr. */ - public String [] addNamedExprArray(@Nullable Collection namedExprs) throws PlanningException { + public String [] addNamedExprArray(@Nullable Collection namedExprs) + throws PlanningException { if (namedExprs != null && namedExprs.size() > 0) { String [] names = new String[namedExprs.size()]; int i = 0; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java index 6e7a5144dc..d818249a2f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java @@ -53,6 +53,7 @@ public interface AlgebraVisitor { RESULT visitAlterTablespace(CONTEXT ctx, Stack stack, AlterTablespace expr) throws PlanningException; RESULT visitAlterTable(CONTEXT ctx, Stack stack, AlterTable expr) throws PlanningException; RESULT visitCreateIndex(CONTEXT ctx, Stack stack, CreateIndex expr) throws PlanningException; + RESULT visitDropIndex(CONTEXT ctx, Stack stack, DropIndex expr) throws PlanningException; RESULT visitTruncateTable(CONTEXT ctx, Stack stack, TruncateTable expr) throws PlanningException; // Insert or Update diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java index ffaf7131e1..664dd80453 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java @@ -128,6 +128,9 @@ public RESULT visit(CONTEXT ctx, Stack stack, Expr expr) throws PlanningEx case TruncateTable: current = visitTruncateTable(ctx, stack, (TruncateTable)expr); break; + case DropIndex: + current = visitDropIndex(ctx, stack, (DropIndex) expr); + break; case Insert: current = visitInsert(ctx, stack, (Insert) expr); @@ -487,6 +490,11 @@ public RESULT visitCreateIndex(CONTEXT ctx, Stack stack, CreateIndex expr) return null; } + @Override + public RESULT visitDropIndex(CONTEXT ctx, Stack stack, DropIndex expr) { + return null; + } + @Override public RESULT visitTruncateTable(CONTEXT ctx, Stack stack, TruncateTable expr) throws PlanningException { return null; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java index 84352f0a65..9e6d78f5c3 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java @@ -18,7 +18,10 @@ package org.apache.tajo.plan.expr; +import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.Column; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import java.util.ArrayList; import java.util.List; @@ -414,4 +417,70 @@ private static void toDisjunctiveNormalFormArrayRecursive(EvalNode node, List { + private Expr expr; + private StringBuilder nameBuilder = new StringBuilder(); + + public IdentifiableNameBuilder(Expr expr) { + this.expr = expr; + } + + public String build() { + Stack stack = new Stack(); + stack.push(expr); + try { + this.visit(null, stack, expr); + } catch (PlanningException e) { + + } + return nameBuilder.deleteCharAt(nameBuilder.length()-1).toString(); + } + + @Override + public Object visitBinaryOperator(Object ctx, Stack stack, BinaryOperator expr) throws PlanningException { + addIntermExpr(expr); + return super.visitBinaryOperator(ctx, stack, expr); + } + + private void append(String str) { + nameBuilder.append(str).append("_"); + } + + private void addIntermExpr(Expr expr) { + this.append(expr.getType().name()); + } + + @Override + public Object visitColumnReference(Object ctx, Stack stack, ColumnReferenceExpr expr) + throws PlanningException { + this.append(expr.getName()); + return super.visitColumnReference(ctx, stack, expr); + } + + @Override + public Object visitLiteral(Object ctx, Stack stack, LiteralValue expr) throws PlanningException { + this.append(expr.getValue()); + return super.visitLiteral(ctx, stack, expr); + } + + @Override + public Object visitNullLiteral(Object ctx, Stack stack, NullLiteral expr) throws PlanningException { + this.append("null"); + return super.visitNullLiteral(ctx, stack, expr); + } + + @Override + public Object visitTimestampLiteral(Object ctx, Stack stack, TimestampLiteral expr) throws PlanningException { + this.append(expr.getDate().toString()); + this.append(expr.getTime().toString()); + return super.visitTimestampLiteral(ctx, stack, expr); + } + + @Override + public Object visitTimeLiteral(Object ctx, Stack stack, TimeLiteral expr) throws PlanningException { + this.append(expr.getTime().toString()); + return super.visitTimeLiteral(ctx, stack, expr); + } + } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java index f5c2cbd637..5b6ebc7291 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java @@ -382,7 +382,7 @@ private static boolean isJoinQualWithOnlyColumns(@Nullable LogicalPlan.QueryBloc return !leftQualifier.equals(rightQualifier); } - static boolean isSingleColumn(EvalNode evalNode) { + public static boolean isSingleColumn(EvalNode evalNode) { return EvalTreeUtil.findUniqueColumns(evalNode).size() == 1; } @@ -517,6 +517,10 @@ public Object visitChild(Object context, EvalNode evalNode, Stack stac return evalNode; } + + public List getEvalNodes() { + return evalNodes; + } } public static class OuterJoinSensitiveEvalFinder extends BasicEvalNodeVisitor { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java index a8d364f713..fcedf48bb3 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java @@ -18,91 +18,99 @@ package org.apache.tajo.plan.logical; -import com.google.common.base.Objects; import com.google.gson.annotations.Expose; -import org.apache.hadoop.fs.Path; +import org.apache.tajo.catalog.IndexMeta; +import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.plan.PlanString; import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.TUtil; + +import java.net.URI; import static org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; public class CreateIndexNode extends UnaryNode implements Cloneable { - @Expose private boolean isUnique; - @Expose private String indexName; - @Expose private Path indexPath; - @Expose private SortSpec[] sortSpecs; - @Expose private IndexMethod indexType = IndexMethod.TWO_LEVEL_BIN_TREE; - @Expose private KeyValueSet options; + @Expose private IndexMeta indexMeta; public CreateIndexNode(int pid) { super(pid, NodeType.CREATE_INDEX); + this.indexMeta = new IndexMeta(); } public void setUnique(boolean unique) { - this.isUnique = unique; + indexMeta.setUnique(unique); } public boolean isUnique() { - return isUnique; + return indexMeta.isUnique(); } public void setIndexName(String indexName) { - this.indexName = indexName; + indexMeta.setIndexName(indexName); } public String getIndexName() { - return this.indexName; + return indexMeta.getIndexName(); } - public void setIndexPath(Path indexPath) { - this.indexPath = indexPath; + public void setIndexPath(URI indexPath) { + indexMeta.setIndexPath(indexPath); } - public Path getIndexPath() { - return this.indexPath; + public URI getIndexPath() { + return indexMeta.getIndexPath(); } - public void setSortSpecs(SortSpec[] sortSpecs) { - this.sortSpecs = sortSpecs; + public void setKeySortSpecs(Schema targetRelationSchema, SortSpec[] sortSpecs) { + indexMeta.setKeySortSpecs(targetRelationSchema, sortSpecs); } - public SortSpec[] getSortSpecs() { - return this.sortSpecs; + public SortSpec[] getKeySortSpecs() { + return indexMeta.getKeySortSpecs(); } - public void setIndexType(IndexMethod indexType) { - this.indexType = indexType; + public void setIndexMethod(IndexMethod indexType) { + indexMeta.setIndexMethod(indexType); } - public IndexMethod getIndexType() { - return this.indexType; + public IndexMethod getIndexMethod() { + return indexMeta.getIndexMethod(); } public void setOptions(KeyValueSet options) { - this.options = options; + indexMeta.setOptions(options); } public KeyValueSet getOptions() { - return this.options; + return indexMeta.getOptions(); + } + + public Schema getTargetRelationSchema() { + return indexMeta.getTargetRelationSchema(); + } + + public boolean hasOptions() { + return indexMeta.getOptions() != null; + } + + public void setClustered(boolean clustered) { + indexMeta.setClustered(clustered); + } + + public boolean isClustered() { + return indexMeta.isClustered(); } @Override public int hashCode() { - return Objects.hashCode(isUnique, indexName, indexPath, sortSpecs, indexType, options); + return indexMeta.hashCode(); } @Override public boolean equals(Object obj) { if (obj instanceof CreateIndexNode) { CreateIndexNode other = (CreateIndexNode) obj; - return this.isUnique == other.isUnique && - TUtil.checkEquals(this.indexName, other.indexName) && - TUtil.checkEquals(this.indexPath, other.indexPath) && - TUtil.checkEquals(this.sortSpecs, other.sortSpecs) && - this.indexType.equals(other.indexType) && - TUtil.checkEquals(this.options, other.options); + return this.indexMeta.equals(other.indexMeta); } return false; } @@ -110,17 +118,13 @@ public boolean equals(Object obj) { @Override public Object clone() throws CloneNotSupportedException { CreateIndexNode createIndexNode = (CreateIndexNode) super.clone(); - createIndexNode.isUnique = isUnique; - createIndexNode.indexName = indexName; - createIndexNode.indexPath = indexPath; - createIndexNode.sortSpecs = sortSpecs.clone(); - createIndexNode.indexType = indexType; - createIndexNode.options = (KeyValueSet) (options != null ? options.clone() : null); + createIndexNode.indexMeta = (IndexMeta) this.indexMeta.clone(); return createIndexNode; } private String getSortSpecString() { StringBuilder sb = new StringBuilder("Column [key= "); + SortSpec[] sortSpecs = indexMeta.getKeySortSpecs(); for (int i = 0; i < sortSpecs.length; i++) { sb.append(sortSpecs[i].getSortKey().getQualifiedName()).append(" ") .append(sortSpecs[i].isAscending() ? "asc" : "desc"); @@ -134,8 +138,9 @@ private String getSortSpecString() { @Override public String toString() { - return "CreateIndex (indexName=" + indexName + ", indexPath=" + indexPath + ", type=" + indexType.name() + - ", isUnique=" + isUnique + ", " + getSortSpecString() + ")"; + return "CreateIndex (indexName=" + indexMeta.getIndexName() + ", indexPath=" + indexMeta.getIndexPath() + + ", type=" + indexMeta.getIndexMethod().name() + + ", isUnique=" + indexMeta.isUnique() + ", " + getSortSpecString() + ")"; } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropIndexNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropIndexNode.java new file mode 100644 index 0000000000..da7018a3cf --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropIndexNode.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.logical; + +import com.google.common.base.Objects; +import org.apache.tajo.plan.PlanString; + +public class DropIndexNode extends LogicalNode implements Cloneable { + private String indexName; + + public DropIndexNode(int pid) { + super(pid, NodeType.DROP_INDEX); + } + + public void init(String indexName) { + this.indexName = indexName; + } + + public int hashCode() { + return Objects.hashCode(indexName); + } + + @Override + public int childNum() { + return 0; + } + + @Override + public LogicalNode getChild(int idx) { + return null; + } + + public boolean equals(Object obj) { + if (obj instanceof DropIndexNode) { + DropIndexNode other = (DropIndexNode) obj; + return super.equals(other) && + this.indexName.equals(other.indexName); + } + return false; + } + + @Override + public void preOrder(LogicalNodeVisitor visitor) { + visitor.visit(this); + } + + @Override + public void postOrder(LogicalNodeVisitor visitor) { + visitor.visit(this); + } + + @Override + public PlanString getPlanString() { + return new PlanString(this); + } + + @Override + public Object clone() throws CloneNotSupportedException { + DropIndexNode clone = (DropIndexNode) super.clone(); + clone.indexName = this.indexName; + return clone; + } + + @Override + public String toString() { + return "DROP INDEX " + indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public String getIndexName() { + return indexName; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java index 0d59733c77..a36e98235b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java @@ -21,56 +21,54 @@ import com.google.gson.Gson; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.datum.Datum; +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; import org.apache.tajo.plan.serder.PlanGsonHelper; +import org.apache.tajo.util.TUtil; + +import java.net.URI; public class IndexScanNode extends ScanNode { - @Expose private SortSpec [] sortKeys; @Expose private Schema keySchema = null; - @Expose private Datum[] datum = null; + @Expose private URI indexPath = null; + @Expose private SimplePredicate[] predicates = null; + + public IndexScanNode(int pid) { + super(pid); + setType(NodeType.INDEX_SCAN); + } public IndexScanNode(int pid, ScanNode scanNode , - Schema keySchema , Datum[] datum, SortSpec[] sortKeys ) { - super(pid); + Schema keySchema , SimplePredicate[] predicates, URI indexPath) { + this(pid); init(scanNode.getTableDesc()); setQual(scanNode.getQual()); setInSchema(scanNode.getInSchema()); setTargets(scanNode.getTargets()); - setType(NodeType.BST_INDEX_SCAN); - this.sortKeys = sortKeys; - this.keySchema = keySchema; - this.datum = datum; + this.set(keySchema, predicates, indexPath); } - - public SortSpec[] getSortKeys() { - return this.sortKeys; + + public void set(Schema keySchema, SimplePredicate[] predicates, URI indexPath) { + this.keySchema = keySchema; + this.indexPath = indexPath; + this.predicates = predicates; } public Schema getKeySchema() { return this.keySchema; } - - public Datum[] getDatum() { - return this.datum; - } - - public void setSortKeys(SortSpec[] sortKeys) { - this.sortKeys = sortKeys; + + public SimplePredicate[] getPredicates() { + return predicates; } - public void setKeySchema( Schema keySchema ) { - this.keySchema = keySchema; - } - @Override public String toString() { Gson gson = PlanGsonHelper.getInstance(); StringBuilder builder = new StringBuilder(); builder.append("IndexScanNode : {\n"); + builder.append(" \"indexPath\" : \"" + gson.toJson(this.indexPath) + "\"\n"); builder.append(" \"keySchema\" : \"" + gson.toJson(this.keySchema) + "\"\n"); - builder.append(" \"sortKeys\" : \"" + gson.toJson(this.sortKeys) + " \"\n"); - builder.append(" \"datums\" : \"" + gson.toJson(this.datum) + "\"\n"); + builder.append(" \"keySortSpecs\" : \"" + gson.toJson(predicates) + " \"\n"); builder.append(" <<\"superClass\" : " + super.toString()); builder.append(">>}"); builder.append("}"); @@ -81,25 +79,12 @@ public String toString() { public boolean equals(Object obj) { if (obj instanceof IndexScanNode) { IndexScanNode other = (IndexScanNode) obj; - boolean eq = super.equals(other); - eq = eq && this.sortKeys.length == other.sortKeys.length; - if(eq) { - for(int i = 0 ; i < this.sortKeys.length ; i ++) { - eq = eq && this.sortKeys[i].getSortKey().equals( - other.sortKeys[i].getSortKey()); - eq = eq && this.sortKeys[i].isAscending() - == other.sortKeys[i].isAscending(); - eq = eq && this.sortKeys[i].isNullFirst() - == other.sortKeys[i].isNullFirst(); - } - } - if(eq) { - for(int i = 0 ; i < this.datum.length ; i ++ ) { - eq = eq && this.datum[i].equals(other.datum[i]); - } - } - return eq; + eq &= this.indexPath.equals(other.indexPath); + eq &= TUtil.checkEquals(this.predicates, other.predicates); + eq &= this.keySchema.equals(other.keySchema); + + return eq; } return false; } @@ -108,15 +93,16 @@ public boolean equals(Object obj) { public Object clone() throws CloneNotSupportedException { IndexScanNode indexNode = (IndexScanNode) super.clone(); indexNode.keySchema = (Schema) this.keySchema.clone(); - indexNode.sortKeys = new SortSpec[this.sortKeys.length]; - for(int i = 0 ; i < sortKeys.length ; i ++ ) - indexNode.sortKeys[i] = (SortSpec) this.sortKeys[i].clone(); - indexNode.datum = new Datum[this.datum.length]; - for(int i = 0 ; i < datum.length ; i ++ ) { - indexNode.datum[i] = this.datum[i]; - } + indexNode.predicates = new SimplePredicate[this.predicates.length]; + for(int i = 0 ; i < this.predicates.length ; i ++ ) + indexNode.predicates[i] = (SimplePredicate) this.predicates[i].clone(); + indexNode.indexPath = this.indexPath; return indexNode; } + + public URI getIndexPath() { + return indexPath; + } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java index c1ff7e170f..1ebea8be46 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java @@ -45,7 +45,7 @@ public enum NodeType { TABLE_SUBQUERY(TableSubQueryNode.class), SCAN(ScanNode.class), PARTITIONS_SCAN(PartitionedTableScanNode.class), - BST_INDEX_SCAN(IndexScanNode.class), + INDEX_SCAN(IndexScanNode.class), STORE(StoreTableNode.class), INSERT(InsertNode.class), @@ -56,6 +56,7 @@ public enum NodeType { ALTER_TABLESPACE (AlterTablespaceNode.class), ALTER_TABLE (AlterTableNode.class), CREATE_INDEX(CreateIndexNode.class), + DROP_INDEX(DropIndexNode.class), TRUNCATE_TABLE (TruncateTableNode.class); private final Class baseClass; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java index 19c254b9a5..6b7e32cca1 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java @@ -20,7 +20,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.OverridableConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; @@ -69,18 +68,19 @@ public void addRewriteRule(LogicalPlanRewriteRule rule) { /** * Rewrite a logical plan with all query rewrite rules added to this engine. * - * @param plan The plan to be rewritten with all query rewrite rule. + * @param context * @return The rewritten plan. */ - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException { LogicalPlanRewriteRule rule; + LogicalPlan plan = null; for (Entry rewriteRule : rewriteRules.entrySet()) { rule = rewriteRule.getValue(); - if (rule.isEligible(queryContext, plan)) { - plan = rule.rewrite(queryContext, plan); + if (rule.isEligible(context)) { if (LOG.isDebugEnabled()) { LOG.debug("The rule \"" + rule.getName() + " \" rewrites the query."); } + plan = rule.rewrite(context); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java index eb96149447..dcfd6bfd8d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java @@ -19,6 +19,7 @@ package org.apache.tajo.plan.rewrite; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.plan.rewrite.rules.AccessPathRewriter; import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule; import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; @@ -52,7 +53,8 @@ public Collection> getPreRules() { public Collection> getPostRules() { List> rules = TUtil.newList( ProjectionPushDownRule.class, - PartitionedTableRewriter.class + PartitionedTableRewriter.class, + AccessPathRewriter.class ); return rules; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java index 267d65134c..f264977ed1 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteEngine.java @@ -18,7 +18,6 @@ package org.apache.tajo.plan.rewrite; -import org.apache.tajo.OverridableConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; @@ -29,5 +28,5 @@ public interface LogicalPlanRewriteEngine { * @param plan The plan to be rewritten with all query rewrite rule. * @return The rewritten plan. */ - LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException; + LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java index 2f0652bd2b..1569a07eeb 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRule.java @@ -18,7 +18,6 @@ package org.apache.tajo.plan.rewrite; -import org.apache.tajo.OverridableConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; @@ -43,7 +42,7 @@ public interface LogicalPlanRewriteRule { * @param plan The plan to be checked * @return True if this rule can be applied to a given plan. Otherwise, false. */ - boolean isEligible(OverridableConf queryContext, LogicalPlan plan); + boolean isEligible(LogicalPlanRewriteRuleContext context); /** * Updates a logical plan and returns an updated logical plan rewritten by this rule. @@ -53,5 +52,5 @@ public interface LogicalPlanRewriteRule { * @param plan Input logical plan. It will not be modified. * @return The rewritten logical plan. */ - LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException; + LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java new file mode 100644 index 0000000000..6c43112e96 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite; + +import org.apache.tajo.OverridableConf; +import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.plan.LogicalPlan; + +public class LogicalPlanRewriteRuleContext { + + private OverridableConf queryContext; + private LogicalPlan plan; + private CatalogService catalog; + + public LogicalPlanRewriteRuleContext(OverridableConf queryContext, LogicalPlan plan) { + setQueryContext(queryContext); + setPlan(plan); + } + + public LogicalPlanRewriteRuleContext(OverridableConf queryContext, LogicalPlan plan, CatalogService catalog) { + setQueryContext(queryContext); + setPlan(plan); + setCatalog(catalog); + } + + public void setCatalog(CatalogService catalog) { + this.catalog = catalog; + } + + public CatalogService getCatalog() { + return catalog; + } + + public OverridableConf getQueryContext() { + return queryContext; + } + + public void setQueryContext(OverridableConf queryContext) { + this.queryContext = queryContext; + } + + public LogicalPlan getPlan() { + return plan; + } + + public void setPlan(LogicalPlan plan) { + this.plan = plan; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathInfo.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathInfo.java new file mode 100644 index 0000000000..477ccafdcd --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathInfo.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite.rules; + +import org.apache.tajo.catalog.statistics.TableStats; + +public abstract class AccessPathInfo { + public enum ScanTypeControl { + INDEX_SCAN, + SEQ_SCAN + } + + private ScanTypeControl scanType; + private TableStats tableStats; + + public AccessPathInfo(ScanTypeControl scanType, TableStats tableStats) { + this.scanType = scanType; + this.tableStats = tableStats; + } + + public ScanTypeControl getScanType() { + return scanType; + } + + public TableStats getTableStats() { + return tableStats; + } + + public void setTableStats(TableStats tableStats) { + this.tableStats = tableStats; + } + + public boolean hasTableStats() { + return this.tableStats != null; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java new file mode 100644 index 0000000000..bfa8d04adb --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/AccessPathRewriter.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite.rules; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.IndexScanNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.RelationNode; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; + +import java.util.List; +import java.util.Stack; + +public class AccessPathRewriter implements LogicalPlanRewriteRule { + private static final Log LOG = LogFactory.getLog(AccessPathRewriter.class); + + private static final String NAME = "Access Path Rewriter"; + private Rewriter rewriter = new Rewriter(); + + @Override + public String getName() { + return NAME; + } + + @Override + public boolean isEligible(LogicalPlanRewriteRuleContext context) { + if (context.getQueryContext().getBool(SessionVars.INDEX_ENABLED)) { + for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) { + for (RelationNode relationNode : block.getRelations()) { + List accessPathInfos = block.getAccessInfos(relationNode); + // If there are any alternative access paths + if (accessPathInfos.size() > 1) { + for (AccessPathInfo accessPathInfo : accessPathInfos) { + if (accessPathInfo.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN) { + return true; + } + } + } + } + } + } + return false; + } + + @Override + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException { + LogicalPlan plan = context.getPlan(); + LogicalPlan.QueryBlock rootBlock = plan.getRootBlock(); + rewriter.init(context.getQueryContext()); + rewriter.visit(rootBlock, plan, rootBlock, rootBlock.getRoot(), new Stack()); + return plan; + } + + private final class Rewriter extends BasicLogicalPlanVisitor { + + private OverridableConf conf; + + public void init(OverridableConf conf) { + this.conf = conf; + } + + @Override + public Object visitScan(Object object, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode, + Stack stack) throws PlanningException { + List accessPaths = block.getAccessInfos(scanNode); + AccessPathInfo optimalPath = null; + // initialize + for (AccessPathInfo accessPath : accessPaths) { + if (accessPath.getScanType() == AccessPathInfo.ScanTypeControl.SEQ_SCAN) { + optimalPath = accessPath; + break; + } + } + // find the optimal path + for (AccessPathInfo accessPath : accessPaths) { + if (accessPath.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN) { + // estimation selectivity and choose the better path + // TODO: improve the selectivity estimation + double estimateSelectivity = 0.001; + double selectivityThreshold = conf.getFloat(SessionVars.INDEX_SELECTIVITY_THRESHOLD); + LOG.info("Selectivity threshold: " + selectivityThreshold); + LOG.info("Estimated selectivity: " + estimateSelectivity); + if (estimateSelectivity < selectivityThreshold) { + // if the estimated selectivity is greater than threshold, use the index scan + optimalPath = accessPath; + } + } + } + + if (optimalPath != null && optimalPath.getScanType() == AccessPathInfo.ScanTypeControl.INDEX_SCAN) { + IndexScanInfo indexScanInfo = (IndexScanInfo) optimalPath; + plan.addHistory("AccessPathRewriter chooses the index scan for " + scanNode.getTableName()); + IndexScanNode indexScanNode = new IndexScanNode(plan.newPID(), scanNode, indexScanInfo.getKeySchema(), + indexScanInfo.getPredicates(), indexScanInfo.getIndexPath()); + if (stack.empty() || block.getRoot().equals(scanNode)) { + block.setRoot(indexScanNode); + } else { + PlannerUtil.replaceNode(plan, stack.peek(), scanNode, indexScanNode); + } + } + return null; + } + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index d7cd82e29a..1ce70191cb 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -23,14 +23,15 @@ import org.apache.commons.logging.LogFactory; import org.apache.tajo.OverridableConf; import org.apache.tajo.algebra.JoinType; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.*; +import org.apache.tajo.datum.Datum; import org.apache.tajo.plan.*; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule.FilterPushDownContext; +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; +import org.apache.tajo.plan.util.IndexUtil; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; @@ -47,6 +48,8 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor pushingDownFilters = new HashSet(); @@ -80,8 +83,8 @@ public String getName() { } @Override - public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { - for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) { + public boolean isEligible(LogicalPlanRewriteRuleContext context) { + for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) { if (block.hasNode(NodeType.SELECTION) || block.hasNode(NodeType.JOIN)) { return true; } @@ -90,7 +93,7 @@ public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { } @Override - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext rewriteRuleContext) throws PlanningException { /* FilterPushDown rule: processing when visits each node - If a target which is corresponding on a filter EvalNode's column is not FieldEval, do not PushDown. @@ -102,6 +105,8 @@ public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throw . It not, create new HavingNode and set parent's child. */ FilterPushDownContext context = new FilterPushDownContext(); + LogicalPlan plan = rewriteRuleContext.getPlan(); + catalog = rewriteRuleContext.getCatalog(); for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) { context.clear(); this.visit(context, plan, block, block.getRoot(), new Stack()); @@ -834,7 +839,7 @@ public LogicalNode visitGroupBy(FilterPushDownContext context, LogicalPlan plan, @Override public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, - LogicalPlan.QueryBlock block, ScanNode scanNode, + LogicalPlan.QueryBlock block, final ScanNode scanNode, Stack stack) throws PlanningException { List matched = Lists.newArrayList(); @@ -904,8 +909,42 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, qual = matched.iterator().next(); } + block.addAccessPath(scanNode, new SeqScanInfo(table)); if (qual != null) { // if a matched qual exists scanNode.setQual(qual); + + // Index path can be identified only after filters are pushed into each scan. + String databaseName, tableName; + databaseName = CatalogUtil.extractQualifier(table.getName()); + tableName = CatalogUtil.extractSimpleName(table.getName()); + Set predicates = TUtil.newHashSet(); + for (EvalNode eval : IndexUtil.getAllEqualEvals(qual)) { + BinaryEval binaryEval = (BinaryEval) eval; + // TODO: consider more complex predicates + if (binaryEval.getLeftExpr().getType() == EvalType.FIELD && + binaryEval.getRightExpr().getType() == EvalType.CONST) { + predicates.add(new Predicate(binaryEval.getType(), + ((FieldEval) binaryEval.getLeftExpr()).getColumnRef(), + ((ConstEval)binaryEval.getRightExpr()).getValue())); + } else if (binaryEval.getLeftExpr().getType() == EvalType.CONST && + binaryEval.getRightExpr().getType() == EvalType.FIELD) { + predicates.add(new Predicate(binaryEval.getType(), + ((FieldEval) binaryEval.getRightExpr()).getColumnRef(), + ((ConstEval)binaryEval.getLeftExpr()).getValue())); + } + } + + // for every subset of the set of columns, find all matched index paths + for (Set subset : Sets.powerSet(predicates)) { + if (subset.size() == 0) + continue; + Column[] columns = extractColumns(subset); + if (catalog.existIndexByColumns(databaseName, tableName, columns)) { + IndexDesc indexDesc = catalog.getIndexByColumns(databaseName, tableName, columns); + block.addAccessPath(scanNode, new IndexScanInfo( + table.getStats(), indexDesc, getSimplePredicates(indexDesc, subset))); + } + } } for (EvalNode matchedEval: matched) { @@ -918,6 +957,49 @@ public LogicalNode visitScan(FilterPushDownContext context, LogicalPlan plan, return scanNode; } + private static class Predicate { + Column column; + Datum value; + EvalType evalType; + + public Predicate(EvalType evalType, Column column, Datum value) { + this.evalType = evalType; + this.column = column; + this.value = value; + } + } + + private static SimplePredicate[] getSimplePredicates(IndexDesc desc, Set predicates) { + SimplePredicate[] simplePredicates = new SimplePredicate[predicates.size()]; + Map colToValue = TUtil.newHashMap(); + for (Predicate predicate : predicates) { + colToValue.put(predicate.column, predicate.value); + } + SortSpec [] keySortSpecs = desc.getKeySortSpecs(); + for (int i = 0; i < keySortSpecs.length; i++) { + simplePredicates[i] = new SimplePredicate(keySortSpecs[i], + colToValue.get(keySortSpecs[i].getSortKey())); + } + return simplePredicates; + } + + private static Datum[] extractPredicateValues(List predicates) { + Datum[] values = new Datum[predicates.size()]; + for (int i = 0; i < values.length; i++) { + values[i] = predicates.get(i).value; + } + return values; + } + + private static Column[] extractColumns(Set predicates) { + Column[] columns = new Column[predicates.size()]; + int i = 0; + for (Predicate p : predicates) { + columns[i++] = p.column; + } + return columns; + } + private void errorFilterPushDown(LogicalPlan plan, LogicalNode node, FilterPushDownContext context) throws PlanningException { String notMatchedNodeStr = ""; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/IndexScanInfo.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/IndexScanInfo.java new file mode 100644 index 0000000000..9ac8ccf8a9 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/IndexScanInfo.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite.rules; + +import com.google.gson.annotations.Expose; +import org.apache.tajo.catalog.IndexDesc; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.common.ProtoObject; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.plan.serder.EvalNodeDeserializer; +import org.apache.tajo.plan.serder.EvalNodeSerializer; +import org.apache.tajo.plan.serder.PlanProto.SimplePredicateProto; + +import java.net.URI; + +public class IndexScanInfo extends AccessPathInfo { + + /** + * Simple predicate represents an equal eval expression which consists of + * a column and a value. + */ + // TODO: extend to represent more complex expressions + public static class SimplePredicate implements ProtoObject { + @Expose private SortSpec keySortSpec; + @Expose private Datum value; + + public SimplePredicate(SortSpec keySortSpec, Datum value) { + this.keySortSpec = keySortSpec; + this.value = value; + } + + public SimplePredicate(SimplePredicateProto proto) { + keySortSpec = new SortSpec(proto.getKeySortSpec()); + value = EvalNodeDeserializer.deserialize(proto.getValue()); + } + + public SortSpec getKeySortSpec() { + return keySortSpec; + } + + public Datum getValue() { + return value; + } + + @Override + public boolean equals(Object o) { + if (o instanceof SimplePredicate) { + SimplePredicate other = (SimplePredicate) o; + return this.keySortSpec.equals(other.keySortSpec) && this.value.equals(other.value); + } else { + return false; + } + } + + @Override + public Object clone() throws CloneNotSupportedException { + SimplePredicate clone = new SimplePredicate(this.keySortSpec, this.value); + return clone; + } + + @Override + public SimplePredicateProto getProto() { + SimplePredicateProto.Builder builder = SimplePredicateProto.newBuilder(); + builder.setKeySortSpec(keySortSpec.getProto()); + builder.setValue(EvalNodeSerializer.serialize(value)); + return builder.build(); + } + } + + private final URI indexPath; + private final Schema keySchema; + private final SimplePredicate[] predicates; + + public IndexScanInfo(TableStats tableStats, IndexDesc indexDesc, SimplePredicate[] predicates) { + super(ScanTypeControl.INDEX_SCAN, tableStats); + this.indexPath = indexDesc.getIndexPath(); + keySchema = new Schema(); + this.predicates = predicates; + for (SimplePredicate predicate : predicates) { + keySchema.addColumn(predicate.getKeySortSpec().getSortKey()); + } + } + + public URI getIndexPath() { + return indexPath; + } + + public Schema getKeySchema() { + return keySchema; + } + + public SimplePredicate[] getPredicates() { + return predicates; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java index 8a24add6c1..c8a81ecb24 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java @@ -23,6 +23,7 @@ import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; import org.apache.tajo.plan.serder.LogicalNodeDeserializer; import org.apache.tajo.plan.serder.LogicalNodeSerializer; import org.apache.tajo.plan.serder.PlanProto; @@ -40,15 +41,16 @@ public String getName() { } @Override - public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { + public boolean isEligible(LogicalPlanRewriteRuleContext context) { return true; } @Override - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException { + LogicalPlan plan = context.getPlan(); LogicalNode root = plan.getRootBlock().getRoot(); PlanProto.LogicalNodeTree serialized = LogicalNodeSerializer.serialize(plan.getRootBlock().getRoot()); - LogicalNode deserialized = LogicalNodeDeserializer.deserialize(queryContext, serialized); + LogicalNode deserialized = LogicalNodeDeserializer.deserialize(context.getQueryContext(), serialized); assert root.deepEquals(deserialized); return plan; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java index 7604c53d9a..962ec1f24e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java @@ -32,6 +32,7 @@ import org.apache.tajo.datum.NullDatum; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.expr.*; @@ -58,8 +59,8 @@ public String getName() { } @Override - public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { - for (LogicalPlan.QueryBlock block : plan.getQueryBlocks()) { + public boolean isEligible(LogicalPlanRewriteRuleContext context) { + for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) { for (RelationNode relation : block.getRelations()) { if (relation.getType() == NodeType.SCAN) { TableDesc table = ((ScanNode)relation).getTableDesc(); @@ -73,9 +74,10 @@ public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { } @Override - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException { + LogicalPlan plan = context.getPlan(); LogicalPlan.QueryBlock rootBlock = plan.getRootBlock(); - rewriter.visit(queryContext, plan, rootBlock, rootBlock.getRoot(), new Stack()); + rewriter.visit(context.getQueryContext(), plan, rootBlock, rootBlock.getRoot(), new Stack()); return plan; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index 64aecf723b..ed46f624bd 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -31,6 +31,7 @@ import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; @@ -56,13 +57,13 @@ public String getName() { } @Override - public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { - LogicalNode toBeOptimized = plan.getRootBlock().getRoot(); + public boolean isEligible(LogicalPlanRewriteRuleContext context) { + LogicalNode toBeOptimized = context.getPlan().getRootBlock().getRoot(); if (PlannerUtil.checkIfDDLPlan(toBeOptimized)) { return false; } - for (QueryBlock eachBlock: plan.getQueryBlocks()) { + for (QueryBlock eachBlock: context.getPlan().getQueryBlocks()) { if (eachBlock.hasTableExpression()) { return true; } @@ -71,7 +72,8 @@ public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { } @Override - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext rewriteRuleContext) throws PlanningException { + LogicalPlan plan = rewriteRuleContext.getPlan(); LogicalPlan.QueryBlock rootBlock = plan.getRootBlock(); LogicalPlan.QueryBlock topmostBlock = rootBlock; @@ -1107,6 +1109,12 @@ public LogicalNode visitPartitionedTableScan(Context context, LogicalPlan plan, return node; } + @Override + public LogicalNode visitIndexScan(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, + IndexScanNode node, Stack stack) throws PlanningException { + return visitScan(context, plan, block,node, stack); + } + @Override public LogicalNode visitTableSubQuery(Context upperContext, LogicalPlan plan, LogicalPlan.QueryBlock block, TableSubQueryNode node, Stack stack) throws PlanningException { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/SeqScanInfo.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/SeqScanInfo.java new file mode 100644 index 0000000000..0c054bdddf --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/SeqScanInfo.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite.rules; + +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.statistics.TableStats; + +public class SeqScanInfo extends AccessPathInfo { + private TableDesc tableDesc; + + public SeqScanInfo(TableStats tableStats) { + super(ScanTypeControl.SEQ_SCAN, tableStats); + } + + public SeqScanInfo(TableDesc tableDesc) { + this(tableDesc.getStats()); + this.setTableDesc(tableDesc); + } + + public TableDesc getTableDesc() { + return tableDesc; + } + + public void setTableDesc(TableDesc tableDesc) { + this.tableDesc = tableDesc; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index 5cbed7ec27..9cdf18b7d8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -36,9 +36,12 @@ import org.apache.tajo.plan.expr.FieldEval; import org.apache.tajo.plan.expr.WindowFunctionEval; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; +import java.net.URI; +import java.net.URISyntaxException; import java.util.*; /** @@ -129,6 +132,9 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { case SCAN: current = convertScan(context, protoNode); break; + case INDEX_SCAN: + current = convertIndexScan(context, protoNode); + break; case CREATE_TABLE: current = convertCreateTable(nodeMap, protoNode); @@ -157,6 +163,13 @@ public int compare(PlanProto.LogicalNode o1, PlanProto.LogicalNode o2) { current = convertTruncateTable(protoNode); break; + case CREATE_INDEX: + current = convertCreateIndex(nodeMap, protoNode); + break; + case DROP_INDEX: + current = convertDropIndex(protoNode); + break; + default: throw new RuntimeException("Unknown NodeType: " + protoNode.getType().name()); } @@ -420,7 +433,24 @@ private static void fillScanNode(OverridableConf context, PlanProto.LogicalNode scan.setOutSchema(convertSchema(protoNode.getOutSchema())); } - private static PartitionedTableScanNode convertPartitionScan(OverridableConf context, PlanProto.LogicalNode protoNode) { + private static IndexScanNode convertIndexScan(OverridableConf context, PlanProto.LogicalNode protoNode) { + IndexScanNode indexScan = new IndexScanNode(protoNode.getNodeId()); + fillScanNode(context, protoNode, indexScan); + + PlanProto.IndexScanSpec indexScanSpec = protoNode.getIndexScan(); + SimplePredicate[] predicates = new SimplePredicate[indexScanSpec.getPredicatesCount()]; + for (int i = 0; i < predicates.length; i++) { + predicates[i] = new SimplePredicate(indexScanSpec.getPredicates(i)); + } + + indexScan.set(new Schema(indexScanSpec.getKeySchema()), predicates, + TUtil.stringToURI(indexScanSpec.getIndexPath())); + + return indexScan; + } + + private static PartitionedTableScanNode convertPartitionScan(OverridableConf context, + PlanProto.LogicalNode protoNode) { PartitionedTableScanNode partitionedScan = new PartitionedTableScanNode(protoNode.getNodeId()); fillScanNode(context, protoNode, partitionedScan); @@ -596,6 +626,45 @@ private static TruncateTableNode convertTruncateTable(PlanProto.LogicalNode prot return truncateTable; } + private static CreateIndexNode convertCreateIndex(Map nodeMap, + PlanProto.LogicalNode protoNode) { + CreateIndexNode createIndex = new CreateIndexNode(protoNode.getNodeId()); + + PlanProto.CreateIndexNode createIndexProto = protoNode.getCreateIndex(); + createIndex.setIndexName(createIndexProto.getIndexName()); + createIndex.setIndexMethod(createIndexProto.getIndexMethod()); + try { + createIndex.setIndexPath(new URI(createIndexProto.getIndexPath())); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + SortSpec[] keySortSpecs = new SortSpec[createIndexProto.getKeySortSpecsCount()]; + for (int i = 0; i < keySortSpecs.length; i++) { + keySortSpecs[i] = new SortSpec(createIndexProto.getKeySortSpecs(i)); + } + createIndex.setKeySortSpecs(new Schema(createIndexProto.getTargetRelationSchema()), + keySortSpecs); + createIndex.setUnique(createIndexProto.getIsUnique()); + createIndex.setClustered(createIndexProto.getIsClustered()); + if (createIndexProto.hasIndexProperties()) { + createIndex.setOptions(new KeyValueSet(createIndexProto.getIndexProperties())); + } + createIndex.setChild(nodeMap.get(createIndexProto.getChildSeq())); + createIndex.setInSchema(convertSchema(protoNode.getInSchema())); + createIndex.setOutSchema(convertSchema(protoNode.getOutSchema())); + + return createIndex; + } + + private static DropIndexNode convertDropIndex(PlanProto.LogicalNode protoNode) { + DropIndexNode dropIndex = new DropIndexNode(protoNode.getNodeId()); + + PlanProto.DropIndexNode dropIndexProto = protoNode.getDropIndex(); + dropIndex.setIndexName(dropIndexProto.getIndexName()); + + return dropIndex; + } + private static AggregationFunctionCallEval [] convertAggFuncCallEvals(OverridableConf context, List evalTrees) { AggregationFunctionCallEval [] aggFuncs = new AggregationFunctionCallEval[evalTrees.size()]; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index 39a13badfb..5ad79c0ef7 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -21,12 +21,15 @@ import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; import org.apache.tajo.algebra.JoinType; +import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; import org.apache.tajo.exception.UnimplementedException; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.Target; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; import org.apache.tajo.plan.serder.PlanProto.AlterTableNode.AddColumn; import org.apache.tajo.plan.serder.PlanProto.AlterTableNode.RenameColumn; import org.apache.tajo.plan.serder.PlanProto.AlterTableNode.RenameTable; @@ -104,6 +107,7 @@ public static class SerializeContext { private LogicalNodeTree.Builder treeBuilder = LogicalNodeTree.newBuilder(); } + @Override public LogicalNode visitRoot(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, LogicalRootNode root, Stack stack) throws PlanningException { super.visitRoot(context, plan, block, root, stack); @@ -138,6 +142,7 @@ public LogicalNode visitSetSession(SerializeContext context, LogicalPlan plan, L return node; } + @Override public LogicalNode visitEvalExpr(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, EvalExprNode exprEval, Stack stack) throws PlanningException { PlanProto.EvalExprNode.Builder exprEvalBuilder = PlanProto.EvalExprNode.newBuilder(); @@ -151,6 +156,7 @@ public LogicalNode visitEvalExpr(SerializeContext context, LogicalPlan plan, Log return exprEval; } + @Override public LogicalNode visitProjection(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ProjectionNode projection, Stack stack) throws PlanningException { super.visitProjection(context, plan, block, projection, stack); @@ -188,6 +194,7 @@ public LogicalNode visitLimit(SerializeContext context, LogicalPlan plan, Logica return limit; } + @Override public LogicalNode visitWindowAgg(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, WindowAggNode windowAgg, Stack stack) throws PlanningException { super.visitWindowAgg(context, plan, block, windowAgg, stack); @@ -262,6 +269,7 @@ public LogicalNode visitHaving(SerializeContext context, LogicalPlan plan, Logic return having; } + @Override public LogicalNode visitGroupBy(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, GroupbyNode node, Stack stack) throws PlanningException { super.visitGroupBy(context, plan, block, node, new Stack()); @@ -297,6 +305,7 @@ private PlanProto.LogicalNode.Builder buildGroupby(SerializeContext context, Gro return nodeBuilder; } + @Override public LogicalNode visitDistinctGroupby(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, DistinctGroupbyNode node, Stack stack) throws PlanningException { super.visitDistinctGroupby(context, plan, block, node, new Stack()); @@ -353,6 +362,7 @@ public LogicalNode visitFilter(SerializeContext context, LogicalPlan plan, Logic return filter; } + @Override public LogicalNode visitJoin(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode join, Stack stack) throws PlanningException { super.visitJoin(context, plan, block, join, stack); @@ -434,6 +444,26 @@ public PlanProto.ScanNode.Builder buildScanNode(ScanNode scan) { return scanBuilder; } + @Override + public LogicalNode visitIndexScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + IndexScanNode node, Stack stack) throws PlanningException { + + PlanProto.ScanNode.Builder scanBuilder = buildScanNode(node); + + PlanProto.IndexScanSpec.Builder indexScanSpecBuilder = PlanProto.IndexScanSpec.newBuilder(); + indexScanSpecBuilder.setKeySchema(node.getKeySchema().getProto()); + indexScanSpecBuilder.setIndexPath(node.getIndexPath().toString()); + for (SimplePredicate predicate : node.getPredicates()) { + indexScanSpecBuilder.addPredicates(predicate.getProto()); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setScan(scanBuilder); + nodeBuilder.setIndexScan(indexScanSpecBuilder); + context.treeBuilder.addNodes(nodeBuilder); + return node; + } + @Override public LogicalNode visitPartitionedTableScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, Stack stack) @@ -458,6 +488,7 @@ public LogicalNode visitPartitionedTableScan(SerializeContext context, LogicalPl return node; } + @Override public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, TableSubQueryNode node, Stack stack) throws PlanningException { super.visitTableSubQuery(context, plan, block, node, stack); @@ -480,6 +511,7 @@ public LogicalNode visitTableSubQuery(SerializeContext context, LogicalPlan plan return node; } + @Override public LogicalNode visitCreateTable(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, CreateTableNode node, Stack stack) throws PlanningException { super.visitCreateTable(context, plan, block, node, stack); @@ -589,6 +621,7 @@ public LogicalNode visitTruncateTable(SerializeContext context, LogicalPlan plan return node; } + @Override public LogicalNode visitInsert(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, InsertNode node, Stack stack) throws PlanningException { super.visitInsert(context, plan, block, node, stack); @@ -670,6 +703,47 @@ public LogicalNode visitDropDatabase(SerializeContext context, LogicalPlan plan, return node; } + @Override + public LogicalNode visitCreateIndex(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + CreateIndexNode node, Stack stack) throws PlanningException { + super.visitCreateIndex(context, plan, block, node, new Stack()); + + PlanProto.CreateIndexNode.Builder createIndexBuilder = PlanProto.CreateIndexNode.newBuilder(); + int [] childIds = registerGetChildIds(context, node); + createIndexBuilder.setChildSeq(childIds[0]); + createIndexBuilder.setIndexName(node.getIndexName()); + createIndexBuilder.setIndexMethod(node.getIndexMethod()); + createIndexBuilder.setIndexPath(node.getIndexPath().toString()); + for (SortSpec sortSpec : node.getKeySortSpecs()) { + createIndexBuilder.addKeySortSpecs(sortSpec.getProto()); + } + createIndexBuilder.setTargetRelationSchema(node.getTargetRelationSchema().getProto()); + createIndexBuilder.setIsUnique(node.isUnique()); + createIndexBuilder.setIsClustered(node.isClustered()); + if (node.hasOptions()) { + createIndexBuilder.setIndexProperties(node.getOptions().getProto()); + } + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setCreateIndex(createIndexBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + + @Override + public LogicalNode visitDropIndex(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + DropIndexNode node, Stack stack) { + PlanProto.DropIndexNode.Builder dropIndexBuilder = PlanProto.DropIndexNode.newBuilder(); + dropIndexBuilder.setIndexName(node.getIndexName()); + + PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); + nodeBuilder.setDropIndex(dropIndexBuilder); + context.treeBuilder.addNodes(nodeBuilder); + + return node; + } + public static PlanProto.NodeType convertType(NodeType type) { return PlanProto.NodeType.valueOf(type.name()); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java new file mode 100644 index 0000000000..73b33d5c0f --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.util; + +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.plan.expr.*; + +import java.util.LinkedList; +import java.util.List; +import java.util.Stack; + +public class IndexUtil { + + public static String getIndexName(String indexName , SortSpec[] keys) { + StringBuilder builder = new StringBuilder(); + builder.append(indexName + "_"); + for(int i = 0 ; i < keys.length ; i ++) { + builder.append(keys[i].getSortKey().getSimpleName() + "_"); + } + return builder.toString(); + } + + public static List getAllEqualEvals(EvalNode qual) { + EvalTreeUtil.EvalFinder finder = new EvalTreeUtil.EvalFinder(EvalType.EQUAL); + finder.visitChild(null, qual, new Stack()); + return finder.getEvalNodes(); + } + + private static class FieldAndValueFinder implements EvalNodeVisitor { + private LinkedList nodeList = new LinkedList(); + + public LinkedList getNodeList () { + return this.nodeList; + } + + @Override + public void visit(EvalNode node) { + BinaryEval binaryEval = (BinaryEval) node; + switch(node.getType()) { + case AND: + break; + case EQUAL: + if( binaryEval.getLeftExpr().getType() == EvalType.FIELD + && binaryEval.getRightExpr().getType() == EvalType.CONST ) { + nodeList.add(binaryEval); + } + break; + case IS_NULL: + if( binaryEval.getLeftExpr().getType() == EvalType.FIELD + && binaryEval.getRightExpr().getType() == EvalType.CONST) { + nodeList.add(binaryEval); + } + } + } + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 0fbd3593ad..ebd47defd9 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -71,10 +71,31 @@ public static boolean checkIfDDLPlan(LogicalNode node) { type == NodeType.CREATE_DATABASE || type == NodeType.DROP_DATABASE || (type == NodeType.CREATE_TABLE && !((CreateTableNode) baseNode).hasSubQuery()) || - baseNode.getType() == NodeType.DROP_TABLE || - baseNode.getType() == NodeType.ALTER_TABLESPACE || - baseNode.getType() == NodeType.ALTER_TABLE || - baseNode.getType() == NodeType.TRUNCATE_TABLE; + type == NodeType.DROP_TABLE || + type == NodeType.ALTER_TABLESPACE || + type == NodeType.ALTER_TABLE || + type == NodeType.TRUNCATE_TABLE || + type == NodeType.CREATE_INDEX || + type == NodeType.DROP_INDEX; + } + + /** + * Most update queries require only the updates to the catalog information, + * but some queries such as "CREATE INDEX" or CTAS requires distributed execution on multiple cluster nodes. + * This function checks whether the given DDL plan requires distributed execution or not. + * @param node the root node of a query plan + * @return Return true if the input query plan requires distributed execution. Otherwise, return false. + */ + public static boolean isDistExecDDL(LogicalNode node) { + LogicalNode baseNode = node; + if (node instanceof LogicalRootNode) { + baseNode = ((LogicalRootNode) node).getChild(); + } + + NodeType type = baseNode.getType(); + + return type == NodeType.CREATE_INDEX || + type == NodeType.CREATE_TABLE && ((CreateTableNode)baseNode).hasSubQuery(); } /** @@ -361,6 +382,12 @@ public LogicalNode visitPartitionedTableScan(ReplacerContext context, LogicalPla throws PlanningException { return node; } + + @Override + public LogicalNode visitIndexScan(ReplacerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, + IndexScanNode node, Stack stack) throws PlanningException { + return node; + } } public static void replaceNode(LogicalNode plan, LogicalNode newNode, NodeType type) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java index 1ff70a233d..9bd3b2403b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java @@ -110,6 +110,9 @@ public RESULT visit(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl case PARTITIONS_SCAN: current = visitPartitionedTableScan(context, plan, block, (PartitionedTableScanNode) node, stack); break; + case INDEX_SCAN: + current = visitIndexScan(context, plan, block, (IndexScanNode) node, stack); + break; case STORE: current = visitStoreTable(context, plan, block, (StoreTableNode) node, stack); break; @@ -140,6 +143,9 @@ public RESULT visit(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl case TRUNCATE_TABLE: current = visitTruncateTable(context, plan, block, (TruncateTableNode) node, stack); break; + case DROP_INDEX: + current = visitDropIndex(context, plan, block, (DropIndexNode) node, stack); + break; default: throw new PlanningException("Unknown logical node type: " + node.getType()); } @@ -310,6 +316,12 @@ public RESULT visitScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBloc return null; } + @Override + public RESULT visitIndexScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, IndexScanNode node, + Stack stack) throws PlanningException { + return null; + } + @Override public RESULT visitPartitionedTableScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, Stack stack) @@ -386,6 +398,12 @@ public RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, LogicalPlan.Qu return result; } + @Override + public RESULT visitDropIndex(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, DropIndexNode node, + Stack stack) { + return null; + } + @Override public RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node, Stack stack) throws PlanningException { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java index 73bc7f13d9..2b33937dbc 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java @@ -240,6 +240,13 @@ public LogicalNode visitCreateIndex(Context context, LogicalPlan plan, LogicalPl return visitUnaryNode(context, plan, block, node, stack); } + @Override + public LogicalNode visitDropIndex(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, + DropIndexNode node, Stack stack) { + context.add(context.depth, node.getPlanString()); + return node; + } + public static String printDepthString(int maxDepth, DepthString planStr) { StringBuilder output = new StringBuilder(); String pad = new String(new char[planStr.getDepth() * 3]).replace('\0', ' '); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java index d0643c90fc..a9fb20bc86 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java @@ -75,6 +75,9 @@ RESULT visitTableSubQuery(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBl RESULT visitScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode node, Stack stack) throws PlanningException; + RESULT visitIndexScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, IndexScanNode node, + Stack stack) throws PlanningException; + RESULT visitPartitionedTableScan(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, Stack stack) throws PlanningException; @@ -105,6 +108,9 @@ RESULT visitAlterTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, CreateIndexNode node, Stack stack) throws PlanningException; + RESULT visitDropIndex(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, DropIndexNode node, + Stack stack) throws PlanningException; + RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node, Stack stack) throws PlanningException; } diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 3e4f07cfcb..dfee969e9e 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -45,7 +45,7 @@ enum NodeType { TABLE_SUBQUERY = 15; SCAN = 16; PARTITIONS_SCAN = 17; - BST_INDEX_SCAN = 18; + INDEX_SCAN = 18; STORE = 19; INSERT = 20; @@ -56,6 +56,8 @@ enum NodeType { ALTER_TABLESPACE = 25; ALTER_TABLE = 26; TRUNCATE_TABLE = 27; + CREATE_INDEX = 28; + DROP_INDEX = 29; } message LogicalNodeTree { @@ -71,31 +73,35 @@ message LogicalNode { optional ScanNode scan = 6; optional PartitionScanSpec partitionScan = 7; - optional JoinNode join = 8; - optional FilterNode filter = 9; - optional GroupbyNode groupby = 10; - optional DistinctGroupbyNode distinctGroupby = 11; - optional SortNode sort = 12; - optional LimitNode limit = 13; - optional WindowAggNode windowAgg = 14; - optional ProjectionNode projection = 15; - optional EvalExprNode exprEval = 16; - optional UnionNode union = 17; - optional TableSubQueryNode tableSubQuery = 18; - optional PersistentStoreNode persistentStore = 19; - optional StoreTableNodeSpec storeTable = 20; - optional InsertNodeSpec insert = 21; - optional CreateTableNodeSpec createTable = 22; - optional RootNode root = 23; - optional SetSessionNode setSession = 24; - - optional CreateDatabaseNode createDatabase = 25; - optional DropDatabaseNode dropDatabase = 26; - optional DropTableNode dropTable = 27; - - optional AlterTablespaceNode alterTablespace = 28; - optional AlterTableNode alterTable = 29; - optional TruncateTableNode truncateTableNode = 30; + optional IndexScanSpec indexScan = 8; + optional JoinNode join = 9; + optional FilterNode filter = 10; + optional GroupbyNode groupby = 11; + optional DistinctGroupbyNode distinctGroupby = 12; + optional SortNode sort = 13; + optional LimitNode limit = 14; + optional WindowAggNode windowAgg = 15; + optional ProjectionNode projection = 16; + optional EvalExprNode exprEval = 17; + optional UnionNode union = 18; + optional TableSubQueryNode tableSubQuery = 19; + optional PersistentStoreNode persistentStore = 20; + optional StoreTableNodeSpec storeTable = 21; + optional InsertNodeSpec insert = 22; + optional CreateTableNodeSpec createTable = 23; + optional RootNode root = 24; + optional SetSessionNode setSession = 25; + + optional CreateDatabaseNode createDatabase = 26; + optional DropDatabaseNode dropDatabase = 27; + optional DropTableNode dropTable = 28; + + optional AlterTablespaceNode alterTablespace = 29; + optional AlterTableNode alterTable = 30; + optional TruncateTableNode truncateTableNode = 31; + + optional CreateIndexNode createIndex = 32; + optional DropIndexNode dropIndex = 33; } message ScanNode { @@ -110,6 +116,17 @@ message PartitionScanSpec { repeated string paths = 1; } +message IndexScanSpec { + required SchemaProto keySchema = 1; + required string indexPath = 2; + repeated SimplePredicateProto predicates = 3; +} + +message SimplePredicateProto { + required SortSpecProto keySortSpec = 1; + required Datum value = 2; +} + message FilterNode { required int32 childSeq = 1; required EvalNodeTree qual = 2; @@ -301,6 +318,22 @@ message AlterTableNode { optional AddColumn addColumn = 5; } +message CreateIndexNode { + required int32 childSeq = 1; + required string indexName = 2; + required IndexMethod indexMethod = 3; + required string indexPath = 4; + repeated SortSpecProto keySortSpecs = 5; + required SchemaProto targetRelationSchema = 6; + optional bool isUnique = 7 [default = false]; + optional bool isClustered = 8 [default = false]; + optional KeyValueSetProto indexProperties = 9; +} + +message DropIndexNode { + required string indexName = 1; +} + enum EvalType { NOT = 0; AND = 1; diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java index e95aeecc93..f9a57fa953 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AddSortForInsertRewriter.java @@ -32,6 +32,7 @@ import org.apache.tajo.plan.logical.SortNode.SortPurpose; import org.apache.tajo.plan.logical.UnaryNode; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; import org.apache.tajo.plan.util.PlannerUtil; public class AddSortForInsertRewriter implements LogicalPlanRewriteRule { @@ -54,13 +55,14 @@ public String getName() { } @Override - public boolean isEligible(OverridableConf queryContext, LogicalPlan plan) { - StoreType storeType = PlannerUtil.getStoreType(plan); + public boolean isEligible(LogicalPlanRewriteRuleContext context) { + StoreType storeType = PlannerUtil.getStoreType(context.getPlan()); return storeType != null; } @Override - public LogicalPlan rewrite(OverridableConf queryContext, LogicalPlan plan) throws PlanningException { + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws PlanningException { + LogicalPlan plan = context.getPlan(); LogicalRootNode rootNode = plan.getRootBlock().getRoot(); UnaryNode insertNode = rootNode.getChild(); LogicalNode childNode = insertNode.getChild(); From de1ba2dabf8ef13b339b645e20c5a1a65814e8ef Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Sun, 1 Mar 2015 23:36:19 +0900 Subject: [PATCH 7/8] TAJO-1301 --- .../org/apache/tajo/algebra/CreateIndex.java | 21 +++++++++- .../apache/tajo/engine/parser/SQLParser.g4 | 4 +- .../tajo/engine/parser/SQLAnalyzer.java | 5 ++- .../apache/tajo/master/exec/DDLExecutor.java | 39 ++++++++++++++++++- .../tajo/master/exec/QueryExecutor.java | 5 +-- .../tajo/engine/query/TestCreateIndex.java | 14 +++++++ .../testCreateIndexOnLocation.sql | 1 + .../org/apache/tajo/plan/LogicalPlanner.java | 9 ++++- .../tajo/plan/logical/CreateIndexNode.java | 17 ++++++-- .../rules/LogicalPlanEqualityTester.java | 1 - .../plan/serder/LogicalNodeDeserializer.java | 1 + .../plan/serder/LogicalNodeSerializer.java | 1 + .../apache/tajo/plan/util/PlannerUtil.java | 2 +- tajo-plan/src/main/proto/Plan.proto | 1 + 14 files changed, 104 insertions(+), 17 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnLocation.sql diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java index 5d9ffa9dc5..a9f734d4f7 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateIndex.java @@ -37,6 +37,9 @@ public class CreateIndex extends UnaryOperator { private Map params; @Expose @SerializedName("IndexMethodSpec") private IndexMethodSpec methodSpec; + private boolean external = false; + @Expose @SerializedName("IndexPath") + private String indexPath; public CreateIndex(final String indexName, final SortSpec[] sortSpecs) { super(OpType.CreateIndex); @@ -85,9 +88,22 @@ public IndexMethodSpec getMethodSpec() { return this.methodSpec; } + public void setIndexPath(String indexPath) { + this.external = true; + this.indexPath = indexPath; + } + + public boolean isExternal() { + return this.external; + } + + public String getIndexPath() { + return this.indexPath; + } + @Override public int hashCode() { - return Objects.hashCode(unique, indexName, sortSpecs, params, methodSpec); + return Objects.hashCode(unique, indexName, sortSpecs, params, methodSpec, external); } @Override @@ -97,7 +113,8 @@ boolean equalsTo(Expr expr) { this.indexName.equals(other.indexName) && TUtil.checkEquals(this.sortSpecs, other.sortSpecs) && TUtil.checkEquals(this.params, other.params) && - this.methodSpec.equals(other.methodSpec); + this.methodSpec.equals(other.methodSpec) && + this.external == other.external; } public static class IndexMethodSpec { diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 6eeeef42d0..22d33b6b23 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -82,8 +82,8 @@ index_statement ; create_index_statement - : CREATE (u=UNIQUE)? INDEX identifier ON table_name (method_specifier)? - LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)? + : CREATE (u=UNIQUE)? INDEX index_name = identifier ON table_name (method_specifier)? + LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)? (LOCATION path=Character_String_Literal)? ; drop_index_statement diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 5cbea861ce..cee9c00384 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1214,7 +1214,7 @@ public Expr visitTrim_function(SQLParser.Trim_functionContext ctx) { @Override public Expr visitCreate_index_statement(SQLParser.Create_index_statementContext ctx) { - String indexName = ctx.identifier().getText(); + String indexName = ctx.index_name.getText(); String tableName = ctx.table_name().getText(); Relation relation = new Relation(tableName); SortSpec[] sortSpecs = buildSortSpecs(ctx.sort_specifier_list()); @@ -1244,6 +1244,9 @@ public Expr visitCreate_index_statement(SQLParser.Create_index_statementContext selection.setChild(relation); projection.setChild(selection); } + if (checkIfExist(ctx.LOCATION())) { + createIndex.setIndexPath(ctx.path.getText()); + } createIndex.setChild(projection); return createIndex; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 9c023a3d6a..dd5451ceea 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -102,8 +102,8 @@ public boolean execute(QueryContext queryContext, LogicalPlan plan) throws IOExc return true; case CREATE_INDEX: - // The catalog information for the created index is automatically updated when the query is successfully finished. - // See the Query.CreateIndexHook class. + CreateIndexNode createIndex = (CreateIndexNode) root; + createIndex(queryContext, createIndex); return true; case DROP_INDEX: @@ -116,6 +116,41 @@ public boolean execute(QueryContext queryContext, LogicalPlan plan) throws IOExc } } + public void createIndex(final QueryContext queryContext, final CreateIndexNode createIndexNode) { + String databaseName, simpleIndexName, qualifiedIndexName; + if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) { + String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName()); + databaseName = splits[0]; + simpleIndexName = splits[1]; + qualifiedIndexName = createIndexNode.getIndexName(); + } else { + databaseName = queryContext.getCurrentDatabase(); + simpleIndexName = createIndexNode.getIndexName(); + qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName); + } + + if (catalog.existIndexByName(databaseName, simpleIndexName)) { + throw new AlreadyExistsIndexException(simpleIndexName); + } + + ScanNode scanNode = PlannerUtil.findTopNode(createIndexNode, NodeType.SCAN); + if (scanNode == null) { + throw new InternalError("Cannot find the table of the relation"); + } + + IndexDesc indexDesc = new IndexDesc(databaseName, scanNode.getTableName(), + simpleIndexName, createIndexNode.getIndexPath(), + createIndexNode.getKeySortSpecs(), createIndexNode.getIndexMethod(), + createIndexNode.isUnique(), false, scanNode.getLogicalSchema()); + + if (catalog.createIndex(indexDesc)) { + LOG.info("Index " + qualifiedIndexName + " is created for the table " + scanNode.getTableName() + "."); + } else { + LOG.info("Index creation " + qualifiedIndexName + " is failed."); + throw new CatalogException("Cannot create index \"" + qualifiedIndexName + "\"."); + } + } + public void dropIndex(final QueryContext queryContext, final DropIndexNode dropIndexNode) { String databaseName, simpleIndexName; if (CatalogUtil.isFQTableName(dropIndexNode.getIndexName())) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index c9ff5542cc..ae3feafd93 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -108,10 +108,9 @@ public SubmitQueryResponse execute(QueryContext queryContext, Session session, S } else { response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); response.setResult(IPCUtil.buildOkRequestResult()); + ddlExecutor.execute(queryContext, plan); } - ddlExecutor.execute(queryContext, plan); - } else if (plan.isExplain()) { // explain query execExplain(plan, response); @@ -139,7 +138,7 @@ public SubmitQueryResponse execute(QueryContext queryContext, Session session, S public void execSetSession(Session session, LogicalPlan plan, SubmitQueryResponse.Builder response) { - SetSessionNode setSessionNode = ((LogicalRootNode)plan.getRootBlock().getRoot()).getChild(); + SetSessionNode setSessionNode = ((LogicalRootNode) plan.getRootBlock().getRoot()).getChild(); final String varName = setSessionNode.getName(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java index 83e33b7ef2..859abee607 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java @@ -95,4 +95,18 @@ public final void testCreateIndexOnMultiExprs() throws Exception { assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx")); assertIndexNotExist(getCurrentDatabase(), "l_orderkey_100_l_linenumber_10_lt10_idx"); } + + @Test + public final void testCreateIndexOnLocation() throws Exception { + executeQuery(); + assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); + assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey"})); + catalog.dropIndex("default", "l_orderkey_idx"); + executeString("create index l_orderkey_idx on lineitem (l_orderkey asc null first) location '/tajo/warehouse/default/l_orderkey_idx';"); + assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); + assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey"})); + executeString("drop index l_orderkey_idx"); + assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); + assertIndexNotExist(getCurrentDatabase(), "l_orderkey_idx"); + } } diff --git a/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnLocation.sql b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnLocation.sql new file mode 100644 index 0000000000..1cb8936b30 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestCreateIndex/testCreateIndexOnLocation.sql @@ -0,0 +1 @@ +create index l_orderkey_idx on lineitem (l_orderkey asc null first); \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index b58f7cb282..2fb28eedc4 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1966,13 +1966,18 @@ public LogicalNode visitCreateIndex(PlanContext context, Stack stack, Crea block.namedExprsMgr.addNamedExprArray(normalizedExprList[i].scalarExprs); } + createIndexNode.setExternal(createIndex.isExternal()); Collection relations = block.getRelations(); assert relations.size() == 1; createIndexNode.setKeySortSpecs(relations.iterator().next().getLogicalSchema(), annotateSortSpecs(block, referNames, sortSpecs)); createIndexNode.setIndexMethod(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase())); - createIndexNode.setIndexPath( - getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName())); + if (createIndex.isExternal()) { + createIndexNode.setIndexPath(new Path(createIndex.getIndexPath()).toUri()); + } else { + createIndexNode.setIndexPath( + getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName())); + } if (createIndex.getParams() != null) { KeyValueSet keyValueSet = new KeyValueSet(); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java index fcedf48bb3..df7eb344ae 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateIndexNode.java @@ -18,6 +18,7 @@ package org.apache.tajo.plan.logical; +import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.IndexMeta; import org.apache.tajo.catalog.Schema; @@ -31,6 +32,7 @@ public class CreateIndexNode extends UnaryNode implements Cloneable { @Expose private IndexMeta indexMeta; + @Expose private boolean external; public CreateIndexNode(int pid) { super(pid, NodeType.CREATE_INDEX); @@ -101,16 +103,24 @@ public boolean isClustered() { return indexMeta.isClustered(); } + public void setExternal(boolean external) { + this.external = external; + } + + public boolean isExternal() { + return this.external; + } + @Override public int hashCode() { - return indexMeta.hashCode(); + return Objects.hashCode(indexMeta, external); } @Override public boolean equals(Object obj) { if (obj instanceof CreateIndexNode) { CreateIndexNode other = (CreateIndexNode) obj; - return this.indexMeta.equals(other.indexMeta); + return this.indexMeta.equals(other.indexMeta) && this.external == other.external; } return false; } @@ -119,6 +129,7 @@ public boolean equals(Object obj) { public Object clone() throws CloneNotSupportedException { CreateIndexNode createIndexNode = (CreateIndexNode) super.clone(); createIndexNode.indexMeta = (IndexMeta) this.indexMeta.clone(); + createIndexNode.external = this.external; return createIndexNode; } @@ -140,7 +151,7 @@ private String getSortSpecString() { public String toString() { return "CreateIndex (indexName=" + indexMeta.getIndexName() + ", indexPath=" + indexMeta.getIndexPath() + ", type=" + indexMeta.getIndexMethod().name() + - ", isUnique=" + indexMeta.isUnique() + ", " + getSortSpecString() + ")"; + ", isUnique=" + indexMeta.isUnique() + ", " + getSortSpecString() + ", isExternal=" + isExternal() + ")"; } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java index c8a81ecb24..35e7a9161a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/LogicalPlanEqualityTester.java @@ -18,7 +18,6 @@ package org.apache.tajo.plan.rewrite.rules; -import org.apache.tajo.OverridableConf; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.logical.LogicalNode; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index 9cdf18b7d8..b70f77956d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -652,6 +652,7 @@ private static CreateIndexNode convertCreateIndex(Map node createIndex.setChild(nodeMap.get(createIndexProto.getChildSeq())); createIndex.setInSchema(convertSchema(protoNode.getInSchema())); createIndex.setOutSchema(convertSchema(protoNode.getOutSchema())); + createIndex.setExternal(createIndexProto.getIsExternal()); return createIndex; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index 5ad79c0ef7..d91db9324c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -723,6 +723,7 @@ public LogicalNode visitCreateIndex(SerializeContext context, LogicalPlan plan, if (node.hasOptions()) { createIndexBuilder.setIndexProperties(node.getOptions().getProto()); } + createIndexBuilder.setIsExternal(node.isExternal()); PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setCreateIndex(createIndexBuilder); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index ebd47defd9..f8e69e87fa 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -94,7 +94,7 @@ public static boolean isDistExecDDL(LogicalNode node) { NodeType type = baseNode.getType(); - return type == NodeType.CREATE_INDEX || + return type == NodeType.CREATE_INDEX && !((CreateIndexNode)baseNode).isExternal() || type == NodeType.CREATE_TABLE && ((CreateTableNode)baseNode).hasSubQuery(); } diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index dfee969e9e..2f2874c48b 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -328,6 +328,7 @@ message CreateIndexNode { optional bool isUnique = 7 [default = false]; optional bool isClustered = 8 [default = false]; optional KeyValueSetProto indexProperties = 9; + optional bool isExternal = 10; } message DropIndexNode { From 8108d010f0090ad713265df2f630f93abf3a3863 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Sun, 1 Mar 2015 23:51:46 +0900 Subject: [PATCH 8/8] TAJO-1301 --- .../main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java | 2 +- .../java/org/apache/tajo/engine/query/TestCreateIndex.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index cee9c00384..11e67d8598 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1245,7 +1245,7 @@ public Expr visitCreate_index_statement(SQLParser.Create_index_statementContext projection.setChild(selection); } if (checkIfExist(ctx.LOCATION())) { - createIndex.setIndexPath(ctx.path.getText()); + createIndex.setIndexPath(stripQuote(ctx.path.getText())); } createIndex.setChild(projection); return createIndex; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java index 859abee607..1a55870827 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java @@ -101,7 +101,8 @@ public final void testCreateIndexOnLocation() throws Exception { executeQuery(); assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey"})); - catalog.dropIndex("default", "l_orderkey_idx"); + catalog.dropIndex(getCurrentDatabase(), "l_orderkey_idx"); + assertFalse(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); executeString("create index l_orderkey_idx on lineitem (l_orderkey asc null first) location '/tajo/warehouse/default/l_orderkey_idx';"); assertTrue(catalog.existIndexByName(getCurrentDatabase(), "l_orderkey_idx")); assertTrue(catalog.existIndexByColumnNames(getCurrentDatabase(), "lineitem", new String[]{"l_orderkey"}));