From f107acd785bfb3df3470f3126b78abe52bb0def1 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 21 Jul 2014 12:34:58 +0900 Subject: [PATCH 1/2] TAJO-837 --- .../org/apache/tajo/algebra/CreateTable.java | 2 +- .../org/apache/tajo/algebra/DropIndex.java | 69 +++++++ .../java/org/apache/tajo/algebra/OpType.java | 1 + .../tajo/catalog/AbstractCatalogClient.java | 42 ----- .../apache/tajo/catalog/CatalogService.java | 5 - .../src/main/proto/CatalogProtocol.proto | 2 - .../java/org/apache/tajo/catalog/Column.java | 1 - .../org/apache/tajo/catalog/IndexDesc.java | 146 +++++++++++--- .../src/main/proto/CatalogProtos.proto | 17 +- .../apache/tajo/catalog/TestIndexDesc.java | 59 ++++-- .../apache/tajo/catalog/CatalogServer.java | 44 ----- .../tajo/catalog/store/AbstractDBStore.java | 178 +++++++----------- .../tajo/catalog/store/CatalogStore.java | 13 +- .../apache/tajo/catalog/store/DerbyStore.java | 6 - .../apache/tajo/catalog/store/MemStore.java | 32 +--- .../main/resources/schemas/derby/indexes.sql | 11 +- .../resources/schemas/mariadb/indexes.sql | 13 +- .../main/resources/schemas/mysql/indexes.sql | 13 +- .../org/apache/tajo/catalog/TestCatalog.java | 31 ++- .../org/apache/tajo/client/TajoClient.java | 49 ++++- tajo-client/src/main/proto/ClientProtos.proto | 4 + .../main/proto/TajoMasterClientProtocol.proto | 4 + .../apache/tajo/engine/parser/SQLParser.g4 | 11 +- .../tajo/engine/parser/SQLAnalyzer.java | 7 +- .../tajo/engine/planner/AlgebraVisitor.java | 1 + .../engine/planner/BaseAlgebraVisitor.java | 8 + .../planner/BasicLogicalPlanVisitor.java | 9 + .../planner/LogicalPlanPreprocessor.java | 7 + .../engine/planner/LogicalPlanVisitor.java | 3 + .../tajo/engine/planner/LogicalPlanner.java | 10 +- .../tajo/engine/planner/PlannerUtil.java | 16 +- .../planner/logical/CreateIndexNode.java | 18 ++ .../engine/planner/logical/DropIndexNode.java | 89 +++++++++ .../tajo/engine/planner/logical/NodeType.java | 1 + .../planner/physical/StoreIndexExec.java | 15 +- .../org/apache/tajo/master/GlobalEngine.java | 169 ++++++++++++++--- .../tajo/master/TajoMasterClientService.java | 42 ++++- .../tajo/worker/TaskAttemptContext.java | 8 - .../org/apache/tajo/QueryTestCaseBase.java | 23 ++- .../engine/planner/TestLogicalPlanner.java | 1 - .../planner/physical/TestPhysicalPlanner.java | 3 +- .../tajo/engine/query/TestCreateIndex.java | 32 +++- 42 files changed, 834 insertions(+), 381 deletions(-) create mode 100644 tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropIndexNode.java diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java index 31217114eb..ad1a6f4a5a 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java @@ -24,9 +24,9 @@ import com.google.gson.annotations.SerializedName; import org.apache.tajo.util.TUtil; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.HashMap; -import java.lang.reflect.Type; import java.util.List; import java.util.Map; diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java new file mode 100644 index 0000000000..f72a12adf2 --- /dev/null +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DropIndex.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.algebra; + +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; +import org.apache.tajo.util.TUtil; + +import java.util.Objects; + +public class DropIndex extends Expr { + + @Expose @SerializedName("IndexName") + private String indexName; + @Expose @SerializedName("IfExists") + private boolean ifExists; + + public DropIndex(String indexName, boolean ifExists) { + super(OpType.DropIndex); + this.indexName = indexName; + this.ifExists = ifExists; + } + + @Override + public int hashCode() { + return Objects.hash(indexName, ifExists); + } + + @Override + boolean equalsTo(Expr expr) { + if (expr instanceof DropIndex) { + DropIndex another = (DropIndex) expr; + return TUtil.checkEquals(indexName, another.indexName) && + ifExists == another.ifExists; + } + return false; + } + + public Object clone() throws CloneNotSupportedException { + DropIndex clone = (DropIndex) super.clone(); + clone.indexName = indexName; + clone.ifExists = ifExists; + return clone; + } + + public String getIndexName() { + return indexName; + } + + public boolean isIfExists() { + return ifExists; + } +} diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java index 2ef22f7246..11c5ad2143 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/OpType.java @@ -51,6 +51,7 @@ public enum OpType { AlterTablespace(AlterTablespace.class), AlterTable(AlterTable.class), CreateIndex(CreateIndex.class), + DropIndex(DropIndex.class), TruncateTable(TruncateTable.class), // Insert or Update diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 542214bfbf..49151cda9c 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -430,26 +430,6 @@ public Boolean call(NettyClientBase client) throws ServiceException { } } - @Override - public boolean existIndexByColumn(final String databaseName, final String tableName, final String columnName) { - try { - return new ServerCallable(this.pool, catalogServerAddr, CatalogProtocol.class, false) { - public Boolean call(NettyClientBase client) throws ServiceException { - - GetIndexByColumnRequest.Builder builder = GetIndexByColumnRequest.newBuilder(); - builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); - builder.setColumnName(columnName); - - CatalogProtocolService.BlockingInterface stub = getStub(client); - return stub.existIndexByColumn(null, builder.build()).getValue(); - } - }.withRetries(); - } catch (ServiceException e) { - LOG.error(e.getMessage(), e); - return false; - } - } - @Override public final IndexDesc getIndexByName(final String databaseName, final String indexName) { try { @@ -470,28 +450,6 @@ public IndexDesc call(NettyClientBase client) throws ServiceException { } } - @Override - public final IndexDesc getIndexByColumn(final String databaseName, - final String tableName, - final String columnName) { - try { - return new ServerCallable(this.pool, catalogServerAddr, CatalogProtocol.class, false) { - public IndexDesc call(NettyClientBase client) throws ServiceException { - - GetIndexByColumnRequest.Builder builder = GetIndexByColumnRequest.newBuilder(); - builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); - builder.setColumnName(columnName); - - CatalogProtocolService.BlockingInterface stub = getStub(client); - return new IndexDesc(stub.getIndexByColumn(null, builder.build())); - } - }.withRetries(); - } catch (ServiceException e) { - LOG.error(e.getMessage(), e); - return null; - } - } - @Override public boolean dropIndex(final String databaseName, final String indexName) { diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java index 667ee88dad..76ab8ba742 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -19,7 +19,6 @@ package org.apache.tajo.catalog; import org.apache.tajo.catalog.partition.PartitionMethodDesc; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes.DataType; import java.util.Collection; @@ -155,12 +154,8 @@ public interface CatalogService { boolean existIndexByName(String databaseName, String indexName); - boolean existIndexByColumn(String databaseName, String tableName, String columnName); - IndexDesc getIndexByName(String databaseName, String indexName); - IndexDesc getIndexByColumn(String databaseName, String tableName, String columnName); - boolean dropIndex(String databaseName, String indexName); boolean createFunction(FunctionDesc funcDesc); diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index c5cb528d34..ee8dbc46df 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -59,9 +59,7 @@ service CatalogProtocolService { rpc createIndex(IndexDescProto) returns (BoolProto); rpc dropIndex(IndexNameProto) returns (BoolProto); rpc existIndexByName(IndexNameProto) returns (BoolProto); - rpc existIndexByColumn(GetIndexByColumnRequest) returns (BoolProto); rpc getIndexByName(IndexNameProto) returns (IndexDescProto); - rpc getIndexByColumn(GetIndexByColumnRequest) returns (IndexDescProto); rpc createFunction(FunctionDescProto) returns (BoolProto); rpc dropFunction(UnregisterFunctionRequest) returns (BoolProto); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java index aceb6f1383..115ee6f989 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java @@ -145,5 +145,4 @@ public String toString() { public String toJson() { return CatalogGsonHelper.toJson(this, Column.class); } - } \ No newline at end of file diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java index cff20593c2..c8c6cd374d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/IndexDesc.java @@ -23,8 +23,12 @@ import com.google.gson.GsonBuilder; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexKeyProto; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.common.ProtoObject; +import org.apache.tajo.util.TUtil; + +import java.util.List; public class IndexDesc implements ProtoObject, Cloneable { private IndexDescProto.Builder builder; @@ -32,35 +36,42 @@ public class IndexDesc implements ProtoObject, Cloneable { private String indexName; // required private String databaseName; // required private String tableName; // required - private Column column; // required private IndexMethod indexMethod; // required + private List keys; private boolean isUnique = false; // optional [default = false] private boolean isClustered = false; // optional [default = false] - private boolean isAscending = false; // optional [default = false] + private String predicate; // optional public IndexDesc() { this.builder = IndexDescProto.newBuilder(); } - public IndexDesc(String idxName, String databaseName, String tableName, Column column, - IndexMethod type, boolean isUnique, boolean isClustered, boolean isAscending) { + public IndexDesc(String idxName, String databaseName, String tableName, IndexMethod type, + List keys, boolean isUnique, boolean isClustered, String predicate) { this(); this.indexName = idxName; this.databaseName = databaseName; this.tableName = tableName; - this.column = column; this.indexMethod = type; + this.keys = keys; this.isUnique = isUnique; this.isClustered = isClustered; - this.isAscending = isAscending; + this.predicate = predicate; } public IndexDesc(IndexDescProto proto) { - this(proto.getIndexName(), - proto.getTableIdentifier().getDatabaseName(), - proto.getTableIdentifier().getTableName(), - new Column(proto.getColumn()), - proto.getIndexMethod(), proto.getIsUnique(), proto.getIsClustered(), proto.getIsAscending()); + this(); + this.indexName = proto.getIndexName(); + this.databaseName = proto.getTableIdentifier().getDatabaseName(); + this.tableName = proto.getTableIdentifier().getTableName(); + this.indexMethod = proto.getMethod(); + this.keys = TUtil.newList(); + for (IndexKeyProto eachProto : proto.getKeysList()) { + this.keys.add(new IndexKey(eachProto)); + } + this.isUnique = proto.getIsUnique(); + this.isClustered = proto.getIsClustered(); + this.predicate = proto.hasPredicate() ? proto.getPredicate() : null; } public String getIndexName() { @@ -71,10 +82,6 @@ public String getTableName() { return tableName; } - public Column getColumn() { - return column; - } - public IndexMethod getIndexMethod() { return this.indexMethod; } @@ -87,14 +94,24 @@ public boolean isUnique() { return this.isUnique; } - public boolean isAscending() { - return this.isAscending; + public List getKeys() { + return keys; + } + + public String getPredicate() { + return predicate; + } + + public boolean hasPredicate() { + return predicate != null; } @Override public IndexDescProto getProto() { if (builder == null) { builder = IndexDescProto.newBuilder(); + } else { + builder.clear(); } CatalogProtos.TableIdentifierProto.Builder tableIdentifierBuilder = CatalogProtos.TableIdentifierProto.newBuilder(); @@ -107,11 +124,15 @@ public IndexDescProto getProto() { builder.setTableIdentifier(tableIdentifierBuilder.build()); builder.setIndexName(this.indexName); - builder.setColumn(this.column.getProto()); - builder.setIndexMethod(indexMethod); + builder.setMethod(indexMethod); builder.setIsUnique(this.isUnique); builder.setIsClustered(this.isClustered); - builder.setIsAscending(this.isAscending); + for (IndexKey eachKey : keys) { + builder.addKeys(eachKey.getProto()); + } + if (this.predicate != null) { + builder.setPredicate(this.predicate); + } return builder.build(); } @@ -121,30 +142,33 @@ public boolean equals(Object obj) { IndexDesc other = (IndexDesc) obj; return getIndexName().equals(other.getIndexName()) && getTableName().equals(other.getTableName()) - && getColumn().equals(other.getColumn()) && getIndexMethod().equals(other.getIndexMethod()) && isUnique() == other.isUnique() && isClustered() == other.isClustered() - && isAscending() == other.isAscending(); + && TUtil.checkEquals(getKeys(), other.getKeys()) + && TUtil.checkEquals(getPredicate(), other.getPredicate()); } else { return false; } } public int hashCode() { - return Objects.hashCode(getIndexName(), getTableName(), getColumn(), - getIndexMethod(), isUnique(), isClustered(), isAscending()); + return Objects.hashCode(getIndexName(), getTableName(), getIndexMethod(), + isUnique(), isClustered(), getKeys(), getPredicate()); } public Object clone() throws CloneNotSupportedException { IndexDesc desc = (IndexDesc) super.clone(); desc.indexName = indexName; desc.tableName = tableName; - desc.column = column; desc.indexMethod = indexMethod; desc.isUnique = isUnique; desc.isClustered = isClustered; - desc.isAscending = isAscending; + desc.keys = TUtil.newList(); + for (IndexKey eachKey : keys) { + desc.keys.add((IndexKey) eachKey.clone()); + } + desc.predicate = predicate; return desc; } @@ -152,4 +176,74 @@ public String toString() { Gson gson = new GsonBuilder().setPrettyPrinting().create(); return gson.toJson(this); } + + public static class IndexKey implements ProtoObject, Cloneable { + private final IndexKeyProto.Builder builder; + private String keyJson; + private boolean ascending = true; + private boolean nullFirst = false; + + public IndexKey() { + builder = IndexKeyProto.newBuilder(); + } + + public IndexKey(final String keyJson, final boolean ascending, final boolean nullFirst) { + this(); + this.keyJson = keyJson; + this.ascending = ascending; + this.nullFirst = nullFirst; + } + + public IndexKey(IndexKeyProto proto) { + this(proto.getKeyJson(), proto.getAscending(), proto.getNullFirst()); + } + + @Override + public IndexKeyProto getProto() { + if (builder != null) { + builder.clear(); + } + builder.setKeyJson(keyJson); + builder.setAscending(ascending); + builder.setNullFirst(nullFirst); + return builder.build(); + } + + @Override + public Object clone() throws CloneNotSupportedException { + IndexKey clone = (IndexKey) super.clone(); + clone.keyJson = this.keyJson; + clone.ascending = this.ascending; + clone.nullFirst = this.nullFirst; + return clone; + } + + @Override + public boolean equals(Object o) { + if (o instanceof IndexKey) { + IndexKey other = (IndexKey) o; + return this.keyJson.equals(other.keyJson) && + this.ascending == other.ascending && + this.nullFirst == other.nullFirst; + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(keyJson, ascending, nullFirst); + } + + public String getKeyJson() { + return keyJson; + } + + public boolean isAscending() { + return ascending; + } + + public boolean isNullFirst() { + return nullFirst; + } + } } \ No newline at end of file diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 367d0b8380..90929a2d45 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -137,14 +137,21 @@ message FunctionDescProto { optional string detail = 8; } +message IndexKeyProto { + required string keyJson = 1; + optional bool ascending = 2 [default = true]; + optional bool nullFirst = 3 [default = false]; +} + message IndexDescProto { - required TableIdentifierProto tableIdentifier = 1; - required string indexName = 2; - required ColumnProto column = 3; - required IndexMethod indexMethod = 4; + required string indexName = 1; + required TableIdentifierProto tableIdentifier = 2; + required IndexMethod method = 3; + repeated IndexKeyProto keys = 4; optional bool isUnique = 5 [default = false]; optional bool isClustered = 6 [default = false]; - optional bool isAscending = 7 [default = false]; + optional string predicate = 7; + // optional string path = 8; } enum IndexMethod { diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java index 3fa8f5399b..6f9f785d4d 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestIndexDesc.java @@ -18,34 +18,55 @@ package org.apache.tajo.catalog; +import org.apache.tajo.catalog.IndexDesc.IndexKey; import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; -import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.util.TUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import java.util.List; + import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.*; public class TestIndexDesc { static IndexDesc desc1; static IndexDesc desc2; static IndexDesc desc3; - + static List keys1; + static List keys2; + static List keys3; + static String pred2; + static { - desc1 = new IndexDesc( - "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), - IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); - - desc2 = new IndexDesc( - "idx_test2", DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), - IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false); - - desc3 = new IndexDesc( - "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), - IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); + keys1 = TUtil.newList(); + keys1.add(new IndexKey("{\"name\":\"id\",\"dataType\":{\"type\":\"INT4\"}}", true, true)); + desc1 = new IndexDesc("idx_test", DEFAULT_DATABASE_NAME, "indexed", + IndexMethod.TWO_LEVEL_BIN_TREE, keys1, true, true, null); + + keys2 = TUtil.newList(); + keys2.add(new IndexKey("{\"name\":\"score\",\"dataType\":{\"type\":\"FLOAT8\"}}", false, false)); + pred2 = "{\n" + + " \"LeftExpr\": {\n" + + " \"ColumnName\": \"score\",\n" + + " \"OpType\": \"Column\"\n" + + " },\n" + + " \"RightExpr\": {\n" + + " \"Value\": \"10\",\n" + + " \"ValueType\": \"Unsigned_Integer\",\n" + + " \"OpType\": \"Literal\"\n" + + " },\n" + + " \"OpType\": \"Equals\"\n" + + "}"; + desc2 = new IndexDesc("idx_test2", DEFAULT_DATABASE_NAME, "indexed", + IndexMethod.TWO_LEVEL_BIN_TREE, keys2, false, false, pred2); + + keys3 = TUtil.newList(); + keys3.add(new IndexKey("{\"name\":\"id\",\"dataType\":{\"type\":\"INT4\"}}", true, true)); + desc3 = new IndexDesc("idx_test", DEFAULT_DATABASE_NAME, "indexed", + IndexMethod.TWO_LEVEL_BIN_TREE, keys3, true, true, null); } @BeforeClass @@ -67,19 +88,19 @@ public void testIndexDescProto() { public void testGetFields() { assertEquals("idx_test", desc1.getIndexName()); assertEquals("indexed", desc1.getTableName()); - assertEquals(new Column("id", Type.INT4), desc1.getColumn()); assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc1.getIndexMethod()); + assertTrue(keys1.equals(desc1.getKeys())); assertEquals(true, desc1.isUnique()); assertEquals(true, desc1.isClustered()); - assertEquals(true, desc1.isAscending()); + assertNull(desc1.getPredicate()); assertEquals("idx_test2", desc2.getIndexName()); assertEquals("indexed", desc2.getTableName()); - assertEquals(new Column("score", Type.FLOAT8), desc2.getColumn()); assertEquals(IndexMethod.TWO_LEVEL_BIN_TREE, desc2.getIndexMethod()); + assertTrue(keys2.equals(desc2.getKeys())); assertEquals(false, desc2.isUnique()); assertEquals(false, desc2.isClustered()); - assertEquals(false, desc2.isAscending()); + assertEquals(pred2, desc2.getPredicate()); } @Test diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 4e391aa71b..55a4745b57 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -738,27 +738,6 @@ public BoolProto existIndexByName(RpcController controller, IndexNameProto reque } } - @Override - public BoolProto existIndexByColumn(RpcController controller, GetIndexByColumnRequest request) - throws ServiceException { - - TableIdentifierProto identifier = request.getTableIdentifier(); - String databaseName = identifier.getDatabaseName(); - String tableName = identifier.getTableName(); - String columnName = request.getColumnName(); - - rlock.lock(); - try { - return store.existIndexByColumn(databaseName, tableName, columnName) ? - ProtoUtil.TRUE : ProtoUtil.FALSE; - } catch (Exception e) { - LOG.error(e); - return BoolProto.newBuilder().setValue(false).build(); - } finally { - rlock.unlock(); - } - } - @Override public IndexDescProto getIndexByName(RpcController controller, IndexNameProto request) throws ServiceException { @@ -780,29 +759,6 @@ public IndexDescProto getIndexByName(RpcController controller, IndexNameProto re } } - @Override - public IndexDescProto getIndexByColumn(RpcController controller, GetIndexByColumnRequest request) - throws ServiceException { - - TableIdentifierProto identifier = request.getTableIdentifier(); - String databaseName = identifier.getDatabaseName(); - String tableName = identifier.getTableName(); - String columnName = request.getColumnName(); - - rlock.lock(); - try { - if (!store.existIndexByColumn(databaseName, tableName, columnName)) { - throw new NoSuchIndexException(databaseName, columnName); - } - return store.getIndexByColumn(databaseName, tableName, columnName); - } catch (Exception e) { - LOG.error("ERROR : cannot get index for " + tableName + "." + columnName, e); - return null; - } finally { - rlock.unlock(); - } - } - @Override public BoolProto dropIndex(RpcController controller, IndexNameProto request) throws ServiceException { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index bcf67741f3..83811bcdfd 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -35,15 +35,11 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.InternalException; import org.apache.tajo.exception.UnimplementedException; -import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.Pair; import java.io.IOException; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.sql.*; import java.util.*; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; @@ -683,20 +679,21 @@ public void createTable(final CatalogProtos.TableDescProto table) throws Catalog pstmt.close(); } - String tidSql = - "SELECT TID from " + TB_TABLES + " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_NAME + "=?"; - pstmt = conn.prepareStatement(tidSql); - pstmt.setInt(1, dbid); - pstmt.setString(2, tableName); - res = pstmt.executeQuery(); - - if (!res.next()) { - throw new CatalogException("ERROR: there is no TID matched to " + table.getTableName()); - } - - int tableId = res.getInt("TID"); - res.close(); - pstmt.close(); +// String tidSql = +// "SELECT TID from " + TB_TABLES + " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_NAME + "=?"; +// pstmt = conn.prepareStatement(tidSql); +// pstmt.setInt(1, dbid); +// pstmt.setString(2, tableName); +// res = pstmt.executeQuery(); +// +// if (!res.next()) { +// throw new CatalogException("ERROR: there is no TID matched to " + table.getTableName()); +// } +// +// int tableId = res.getInt("TID"); + int tableId = getTableId(dbid, databaseName, tableName); +// res.close(); +// pstmt.close(); String colSql = "INSERT INTO " + TB_COLUMNS + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?) "; @@ -1610,7 +1607,6 @@ public void createIndex(final IndexDescProto proto) throws CatalogException { String databaseName = proto.getTableIdentifier().getDatabaseName(); String tableName = proto.getTableIdentifier().getTableName(); - String columnName = CatalogUtil.extractSimpleName(proto.getColumn().getName()); try { int databaseId = getDatabaseId(databaseName); @@ -1618,12 +1614,25 @@ public void createIndex(final IndexDescProto proto) throws CatalogException { String sql = "INSERT INTO " + TB_INDEXES + " (" + COL_DATABASES_PK + ", " + COL_TABLES_PK + ", INDEX_NAME, " + - "COLUMN_NAME, DATA_TYPE, INDEX_TYPE, IS_UNIQUE, IS_CLUSTERED, IS_ASCENDING) VALUES (?,?,?,?,?,?,?,?,?)"; + "METHOD, EXPR_NUM, EXPRS, ASC_ORDERS, NULL_ORDERS, IS_UNIQUE, IS_CLUSTERED, PRED) " + + "VALUES (?,?,?,?,?,?,?,?,?,?,?)"; if (LOG.isDebugEnabled()) { LOG.debug(sql); } + StringBuilder exprsBuilder = new StringBuilder(); + StringBuilder ascOrdersBuilder = new StringBuilder(); + StringBuilder nullOrdersBuilder = new StringBuilder(); + for (IndexKeyProto eachKey : proto.getKeysList()) { + exprsBuilder.append(eachKey.getKeyJson()).append(","); + ascOrdersBuilder.append(eachKey.getAscending()).append(","); + nullOrdersBuilder.append(eachKey.getNullFirst()).append(","); + } + exprsBuilder.deleteCharAt(exprsBuilder.length()-1); + ascOrdersBuilder.deleteCharAt(ascOrdersBuilder.length()-1); + nullOrdersBuilder.deleteCharAt(nullOrdersBuilder.length()-1); + conn = getConnection(); conn.setAutoCommit(false); @@ -1631,12 +1640,18 @@ public void createIndex(final IndexDescProto proto) throws CatalogException { pstmt.setInt(1, databaseId); pstmt.setInt(2, tableId); pstmt.setString(3, proto.getIndexName()); - pstmt.setString(4, columnName); - pstmt.setString(5, proto.getColumn().getDataType().getType().name()); - pstmt.setString(6, proto.getIndexMethod().toString()); - pstmt.setBoolean(7, proto.hasIsUnique() && proto.getIsUnique()); - pstmt.setBoolean(8, proto.hasIsClustered() && proto.getIsClustered()); - pstmt.setBoolean(9, proto.hasIsAscending() && proto.getIsAscending()); + pstmt.setString(4, proto.getMethod().name()); + pstmt.setInt(5, proto.getKeysCount()); + pstmt.setString(6, exprsBuilder.toString()); + pstmt.setString(7, ascOrdersBuilder.toString()); + pstmt.setString(8, nullOrdersBuilder.toString()); + pstmt.setBoolean(9, proto.hasIsUnique() && proto.getIsUnique()); + pstmt.setBoolean(10, proto.hasIsClustered() && proto.getIsClustered()); + if (proto.hasPredicate()) { + pstmt.setString(11, proto.getPredicate()); + } else { + pstmt.setNull(11, Types.VARCHAR, "PRED"); + } pstmt.executeUpdate(); conn.commit(); } catch (SQLException se) { @@ -1649,23 +1664,38 @@ public void createIndex(final IndexDescProto proto) throws CatalogException { @Override public void dropIndex(String databaseName, final String indexName) throws CatalogException { Connection conn = null; - PreparedStatement pstmt = null; try { - int databaseId = getDatabaseId(databaseName); - String sql = "DELETE FROM " + TB_INDEXES + " WHERE " + COL_DATABASES_PK + "=? AND INDEX_NAME=?"; + conn = getConnection(); + conn.setAutoCommit(false); + dropIndexInternal(conn, databaseName, indexName); + conn.commit(); + } catch (SQLException se) { + try { + conn.rollback(); + } catch (SQLException e) { + LOG.error(e); + } + } finally { + CatalogUtil.closeQuietly(conn); + } + } + private void dropIndexInternal(Connection conn, final String databaseName, final String indexName) + throws SQLException { + PreparedStatement pstmt = null; + try { + int databaseId = getDatabaseId(databaseName); + String sql = "DELETE FROM " + TB_INDEXES + + " WHERE " + COL_DATABASES_PK + "=? AND INDEX_NAME=?"; if (LOG.isDebugEnabled()) { LOG.debug(sql); } - conn = getConnection(); pstmt = conn.prepareStatement(sql); pstmt.setInt(1, databaseId); pstmt.setString(2, indexName); pstmt.executeUpdate(); - } catch (SQLException se) { - throw new CatalogException(se); } finally { CatalogUtil.closeQuietly(pstmt); } @@ -1690,8 +1720,8 @@ public static String getTableName(Connection conn, int tableId) throws SQLExcept } final static String GET_INDEXES_SQL = - "SELECT " + COL_TABLES_PK + ", INDEX_NAME, COLUMN_NAME, DATA_TYPE, INDEX_TYPE, IS_UNIQUE, " + - "IS_CLUSTERED, IS_ASCENDING FROM " + TB_INDEXES; + "SELECT " + COL_TABLES_PK + ", INDEX_NAME, EXPRS, METHOD, IS_UNIQUE, " + + "IS_CLUSTERED FROM " + TB_INDEXES; @Override @@ -1733,46 +1763,6 @@ public IndexDescProto getIndexByName(String databaseName, final String indexName return proto; } - @Override - public IndexDescProto getIndexByColumn(final String databaseName, - final String tableName, - final String columnName) throws CatalogException { - Connection conn = null; - ResultSet res = null; - PreparedStatement pstmt = null; - IndexDescProto proto = null; - - try { - int databaseId = getDatabaseId(databaseName); - - String sql = GET_INDEXES_SQL + " WHERE " + COL_DATABASES_PK + "=? AND COLUMN_NAME=?"; - - if (LOG.isDebugEnabled()) { - LOG.debug(sql); - } - - conn = getConnection(); - pstmt = conn.prepareStatement(sql); - pstmt.setInt(1, databaseId); - ; - pstmt.setString(2, columnName); - res = pstmt.executeQuery(); - if (!res.next()) { - throw new CatalogException("ERROR: there is no index matched to " + columnName); - } - IndexDescProto.Builder builder = IndexDescProto.newBuilder(); - resultToIndexDescProtoBuilder(builder, res); - builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); - proto = builder.build(); - } catch (SQLException se) { - throw new CatalogException(se); - } finally { - CatalogUtil.closeQuietly(pstmt, res); - } - - return proto; - } - @Override public boolean existIndexByName(String databaseName, final String indexName) throws CatalogException { Connection conn = null; @@ -1806,39 +1796,6 @@ public boolean existIndexByName(String databaseName, final String indexName) thr return exist; } - @Override - public boolean existIndexByColumn(String databaseName, String tableName, String columnName) - throws CatalogException { - Connection conn = null; - ResultSet res = null; - PreparedStatement pstmt = null; - - boolean exist = false; - - try { - int databaseId = getDatabaseId(databaseName); - - String sql = - "SELECT INDEX_NAME FROM " + TB_INDEXES + " WHERE " + COL_DATABASES_PK + "=? AND COLUMN_NAME=?"; - - if (LOG.isDebugEnabled()) { - LOG.debug(sql); - } - - conn = getConnection(); - pstmt = conn.prepareStatement(sql); - pstmt.setInt(1, databaseId); - pstmt.setString(2, columnName); - res = pstmt.executeQuery(); - exist = res.next(); - } catch (SQLException se) { - throw new CatalogException(se); - } finally { - CatalogUtil.closeQuietly(pstmt, res); - } - return exist; - } - @Override public IndexDescProto[] getIndexes(String databaseName, final String tableName) throws CatalogException { @@ -1884,11 +1841,10 @@ public IndexDescProto[] getIndexes(String databaseName, final String tableName) private void resultToIndexDescProtoBuilder(IndexDescProto.Builder builder, final ResultSet res) throws SQLException { builder.setIndexName(res.getString("index_name")); - builder.setColumn(indexResultToColumnProto(res)); - builder.setIndexMethod(getIndexMethod(res.getString("index_type").trim())); +// builder.setkeys(indexResultToColumnProto(res)); + builder.setMethod(getIndexMethod(res.getString("method").trim())); builder.setIsUnique(res.getBoolean("is_unique")); builder.setIsClustered(res.getBoolean("is_clustered")); - builder.setIsAscending(res.getBoolean("is_ascending")); } /** diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 5de96330de..46fab64512 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -19,12 +19,11 @@ package org.apache.tajo.catalog.store; import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.IndexDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.*; import java.io.Closeable; -import org.apache.tajo.catalog.exception.CatalogException; - import java.util.Collection; import java.util.List; @@ -106,13 +105,13 @@ void addPartition(String databaseName, String tableName, IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException; - IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName) - throws CatalogException; +// IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName) +// throws CatalogException; boolean existIndexByName(String databaseName, String indexName) throws CatalogException; - boolean existIndexByColumn(String databaseName, String tableName, String columnName) - throws CatalogException; +// boolean existIndexByColumn(String databaseName, String tableName, String columnName) +// throws CatalogException; IndexDescProto [] getIndexes(String databaseName, String tableName) throws CatalogException; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java index d6f9fc39cd..6a3b13cab8 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/DerbyStore.java @@ -204,12 +204,6 @@ protected void createBaseTable() throws CatalogException { } stmt.addBatch(sql); - sql = "CREATE INDEX idx_indexes_columns ON " + TB_INDEXES + "(" + COL_DATABASES_PK + ",column_name)"; - - if (LOG.isDebugEnabled()) { - LOG.debug(sql); - } - stmt.addBatch(sql); stmt.executeBatch(); LOG.info("Table '" + TB_INDEXES + "' is created."); baseTableMaps.put(TB_INDEXES, true); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index ca9916005c..db9b8f230c 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -120,6 +120,8 @@ public void createDatabase(String databaseName, String tablespaceName) throws Ca } databases.put(databaseName, new HashMap()); + indexes.put(databaseName, new HashMap()); + indexesByColumn.put(databaseName, new HashMap()); } @Override @@ -368,9 +370,11 @@ public void createIndex(IndexDescProto proto) throws CatalogException { throw new AlreadyExistsIndexException(proto.getIndexName()); } - index.put(proto.getIndexName(), proto); - indexByColumn.put(proto.getTableIdentifier().getTableName() + "." - + CatalogUtil.extractSimpleName(proto.getColumn().getName()), proto); + String indexName = proto.getIndexName(); + if (CatalogUtil.isFQTableName(indexName)) { + indexName = CatalogUtil.splitFQTableName(indexName)[1]; + } + index.put(indexName, proto); } /* (non-Javadoc) @@ -398,34 +402,12 @@ public IndexDescProto getIndexByName(String databaseName, String indexName) thro return index.get(indexName); } - /* (non-Javadoc) - * @see CatalogStore#getIndexByName(java.lang.String, java.lang.String) - */ - @Override - public IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName) - throws CatalogException { - - Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - if (!indexByColumn.containsKey(columnName)) { - throw new NoSuchIndexException(columnName); - } - - return indexByColumn.get(columnName); - } - @Override public boolean existIndexByName(String databaseName, String indexName) throws CatalogException { Map index = checkAndGetDatabaseNS(indexes, databaseName); return index.containsKey(indexName); } - @Override - public boolean existIndexByColumn(String databaseName, String tableName, String columnName) - throws CatalogException { - Map indexByColumn = checkAndGetDatabaseNS(indexesByColumn, databaseName); - return indexByColumn.containsKey(columnName); - } - @Override public IndexDescProto[] getIndexes(String databaseName, String tableName) throws CatalogException { List protos = new ArrayList(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql index c4cfc256d1..45ed1441b8 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/indexes.sql @@ -1,12 +1,15 @@ CREATE TABLE INDEXES ( +-- INDEX_ID INT NOT NULL GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1), DB_ID INT NOT NULL REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE, TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE, INDEX_NAME VARCHAR(128) NOT NULL, - COLUMN_NAME VARCHAR(128) NOT NULL, - DATA_TYPE VARCHAR(128) NOT NULL, - INDEX_TYPE CHAR(32) NOT NULL, + METHOD CHAR(32) NOT NULL, + EXPR_NUM INT NOT NULL, + EXPRS VARCHAR(1024) NOT NULL, + ASC_ORDERS VARCHAR(1024) NOT NULL, + NULL_ORDERS VARCHAR(1024) NOT NULL, IS_UNIQUE BOOLEAN NOT NULL, IS_CLUSTERED BOOLEAN NOT NULL, - IS_ASCENDING BOOLEAN NOT NULL, + PRED VARCHAR(4096), CONSTRAINT C_INDEXES_PK PRIMARY KEY (DB_ID, INDEX_NAME) ) \ No newline at end of file diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql index 62feb3630e..b375a48c18 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/indexes.sql @@ -1,16 +1,17 @@ CREATE TABLE INDEXES ( +-- INDEX_ID INT NOT NULL AUTO_INCREMENT PRIMARY KEY, DB_ID INT NOT NULL, TID INT NOT NULL, INDEX_NAME VARCHAR(128) NOT NULL, - COLUMN_NAME VARCHAR(128) NOT NULL, - DATA_TYPE VARCHAR(128) NOT NULL, - INDEX_TYPE CHAR(32) NOT NULL, + METHOD CHAR(32) NOT NULL, + EXPR_NUM INT NOT NULL, + EXPRS VARCHAR(1024) NOT NULL, + ASC_ORDERS VARCHAR(1024) NOT NULL, + NULL_ORDERS VARCHAR(1024) NOT NULL, IS_UNIQUE BOOLEAN NOT NULL, IS_CLUSTERED BOOLEAN NOT NULL, - IS_ASCENDING BOOLEAN NOT NULL, - PRIMARY KEY (DB_ID, INDEX_NAME), + PRED VARCHAR(4096), FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE, FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE, UNIQUE INDEX IDX_DB_ID_NAME (DB_ID, INDEX_NAME), - INDEX IDX_TID_COLUMN_NAME (TID, COLUMN_NAME) ) \ No newline at end of file diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql index 62feb3630e..b375a48c18 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/indexes.sql @@ -1,16 +1,17 @@ CREATE TABLE INDEXES ( +-- INDEX_ID INT NOT NULL AUTO_INCREMENT PRIMARY KEY, DB_ID INT NOT NULL, TID INT NOT NULL, INDEX_NAME VARCHAR(128) NOT NULL, - COLUMN_NAME VARCHAR(128) NOT NULL, - DATA_TYPE VARCHAR(128) NOT NULL, - INDEX_TYPE CHAR(32) NOT NULL, + METHOD CHAR(32) NOT NULL, + EXPR_NUM INT NOT NULL, + EXPRS VARCHAR(1024) NOT NULL, + ASC_ORDERS VARCHAR(1024) NOT NULL, + NULL_ORDERS VARCHAR(1024) NOT NULL, IS_UNIQUE BOOLEAN NOT NULL, IS_CLUSTERED BOOLEAN NOT NULL, - IS_ASCENDING BOOLEAN NOT NULL, - PRIMARY KEY (DB_ID, INDEX_NAME), + PRED VARCHAR(4096), FOREIGN KEY (DB_ID) REFERENCES DATABASES_ (DB_ID) ON DELETE CASCADE, FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE, UNIQUE INDEX IDX_DB_ID_NAME (DB_ID, INDEX_NAME), - INDEX IDX_TID_COLUMN_NAME (TID, COLUMN_NAME) ) \ No newline at end of file diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 4039ff2e1f..9f15aae4ea 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -21,6 +21,7 @@ import com.google.common.collect.Sets; import org.apache.hadoop.fs.Path; import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.IndexDesc.IndexKey; import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.function.Function; @@ -30,8 +31,8 @@ import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.store.DerbyStore; -import org.apache.tajo.catalog.store.MySQLStore; import org.apache.tajo.catalog.store.MariaDBStore; +import org.apache.tajo.catalog.store.MySQLStore; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -51,7 +52,6 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; public class TestCatalog { static final String FieldName1="f1"; @@ -373,17 +373,20 @@ public void testGetTable() throws Exception { static IndexDesc desc3; static { - desc1 = new IndexDesc( - "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), - IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); + List indexKeys = new ArrayList(); + indexKeys.add(new IndexKey(new Column("id", Type.INT4).toJson(), true, true)); + desc1 = new IndexDesc("idx_test", DEFAULT_DATABASE_NAME, "indexed", IndexMethod.TWO_LEVEL_BIN_TREE, + indexKeys, true, true, null); - desc2 = new IndexDesc( - "idx_test2", DEFAULT_DATABASE_NAME, "indexed", new Column("score", Type.FLOAT8), - IndexMethod.TWO_LEVEL_BIN_TREE, false, false, false); + indexKeys = new ArrayList(); + indexKeys.add(new IndexKey(new Column("score", Type.FLOAT8).toJson(), false, false)); + desc2 = new IndexDesc("idx_test2", DEFAULT_DATABASE_NAME, "indexed", IndexMethod.TWO_LEVEL_BIN_TREE, + indexKeys, false, false, null); - desc3 = new IndexDesc( - "idx_test", DEFAULT_DATABASE_NAME, "indexed", new Column("id", Type.INT4), - IndexMethod.TWO_LEVEL_BIN_TREE, true, true, true); + indexKeys = new ArrayList(); + indexKeys.add(new IndexKey(new Column("id", Type.INT4).toJson(), true, true)); + desc3 = new IndexDesc("idx_test", DEFAULT_DATABASE_NAME, "indexed", IndexMethod.TWO_LEVEL_BIN_TREE, + indexKeys, true, true, null); } public static TableDesc prepareTable() throws IOException { @@ -439,18 +442,14 @@ public void testAddAndDelIndex() throws Exception { assertTrue(catalog.createTable(desc)); assertFalse(catalog.existIndexByName("db1", desc1.getIndexName())); - assertFalse(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "id")); catalog.createIndex(desc1); assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getIndexName())); - assertTrue(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "id")); assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getIndexName())); - assertFalse(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "score")); catalog.createIndex(desc2); assertTrue(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc2.getIndexName())); - assertTrue(catalog.existIndexByColumn(DEFAULT_DATABASE_NAME, "indexed", "score")); - + catalog.dropIndex(DEFAULT_DATABASE_NAME, desc1.getIndexName()); assertFalse(catalog.existIndexByName(DEFAULT_DATABASE_NAME, desc1.getIndexName())); catalog.dropIndex(DEFAULT_DATABASE_NAME, desc2.getIndexName()); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java index 333c8d6541..b466b96a2e 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java @@ -29,7 +29,10 @@ import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.annotation.ThreadSafe; -import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.cli.InvalidClientSessionException; import org.apache.tajo.conf.TajoConf; @@ -762,6 +765,50 @@ public Boolean call(NettyClientBase client) throws ServiceException { } + /** + * Drop an index + * + * @param indexName The index name to be dropped. This name is case sensitive. + * @return True if the index is dropped successfully. + */ + public boolean dropIndex(final String indexName) throws ServiceException { + return new ServerCallable(connPool, tajoMasterAddr, + TajoMasterClientProtocol.class, false, true) { + public Boolean call(NettyClientBase client) throws ServiceException { + checkSessionAndGet(client); + + TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); + + SessionedStringProto.Builder builder = SessionedStringProto.newBuilder(); + builder.setSessionId(sessionId); + builder.setValue(indexName); + return tajoMasterService.dropIndex(null, builder.build()).getValue(); + } + }.withRetries(); + } + + /** + * Does the index exist? + * + * @param indexName The index name to be checked. This name is case sensitive. + * @return True if so. + */ + public boolean existIndex(final String indexName) throws ServiceException { + return new ServerCallable(connPool, tajoMasterAddr, + TajoMasterClientProtocol.class, false, true) { + public Boolean call(NettyClientBase client) throws ServiceException { + checkSessionAndGet(client); + + TajoMasterClientProtocolService.BlockingInterface tajoMasterService = client.getStub(); + + SessionedStringProto.Builder builder = SessionedStringProto.newBuilder(); + builder.setSessionId(sessionId); + builder.setValue(indexName); + return tajoMasterService.existIndex(null, builder.build()).getValue(); + } + }.withRetries(); + } + public List getRunningQueryList() throws ServiceException { return new ServerCallable>(connPool, tajoMasterAddr, TajoMasterClientProtocol.class, false, true) { diff --git a/tajo-client/src/main/proto/ClientProtos.proto b/tajo-client/src/main/proto/ClientProtos.proto index c66b228538..f56383e473 100644 --- a/tajo-client/src/main/proto/ClientProtos.proto +++ b/tajo-client/src/main/proto/ClientProtos.proto @@ -224,3 +224,7 @@ message FunctionResponse { repeated FunctionDescProto functions = 2; optional string errorMessage = 3; } + +message CreateIndexRequest { + required IndexDescProto desc = 1; +} \ No newline at end of file diff --git a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto index 9495fb15dd..79f7c273d0 100644 --- a/tajo-client/src/main/proto/TajoMasterClientProtocol.proto +++ b/tajo-client/src/main/proto/TajoMasterClientProtocol.proto @@ -65,4 +65,8 @@ service TajoMasterClientProtocolService { rpc getTableList(GetTableListRequest) returns (GetTableListResponse); rpc getTableDesc(GetTableDescRequest) returns (TableResponse); rpc getFunctionList(SessionedStringProto) returns (FunctionResponse); + + // Index Management APIs + rpc existIndex(SessionedStringProto) returns (BoolProto); + rpc dropIndex(SessionedStringProto) returns (BoolProto); } diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 706c9738fb..438a255451 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -68,10 +68,19 @@ schema_statement ; index_statement - : CREATE (u=UNIQUE)? INDEX identifier ON table_name (method_specifier)? + : create_index_statement + | drop_index_statement + ; + +create_index_statement + : CREATE (u=UNIQUE)? INDEX (if_not_exists)? identifier ON table_name (method_specifier)? LEFT_PAREN sort_specifier_list RIGHT_PAREN param_clause? (where_clause)? ; +drop_index_statement + : DROP INDEX (if_exists)? index_name = identifier + ; + database_definition : CREATE DATABASE (if_not_exists)? dbname = identifier ; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index faf0212a38..08f79398af 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1142,7 +1142,7 @@ public Expr visitTrim_function(SQLParser.Trim_functionContext ctx) { } @Override - public Expr visitIndex_statement(SQLParser.Index_statementContext ctx) { + public Expr visitCreate_index_statement(@NotNull SQLParser.Create_index_statementContext ctx) { String indexName = ctx.identifier().getText(); String tableName = ctx.table_name().getText(); Relation relation = new Relation(tableName); @@ -1177,6 +1177,11 @@ public Expr visitIndex_statement(SQLParser.Index_statementContext ctx) { return createIndex; } + @Override + public Expr visitDrop_index_statement(@NotNull SQLParser.Drop_index_statementContext ctx) { + return new DropIndex(ctx.index_name.getText(), checkIfExist(ctx.if_exists())); + } + @Override public Expr visitDatabase_definition(@NotNull SQLParser.Database_definitionContext ctx) { return new CreateDatabase(ctx.identifier().getText(), null, checkIfExist(ctx.if_not_exists())); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java index eb8a21f106..0d73dfc577 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java @@ -49,6 +49,7 @@ public interface AlgebraVisitor { RESULT visitAlterTablespace(CONTEXT ctx, Stack stack, AlterTablespace expr) throws PlanningException; RESULT visitAlterTable(CONTEXT ctx, Stack stack, AlterTable expr) throws PlanningException; RESULT visitCreateIndex(CONTEXT ctx, Stack stack, CreateIndex expr) throws PlanningException; + RESULT visitDropIndex(CONTEXT ctx, Stack stack, DropIndex expr) throws PlanningException; RESULT visitTruncateTable(CONTEXT ctx, Stack stack, TruncateTable expr) throws PlanningException; // Insert or Update diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java index b93541b714..4c9932aaed 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java @@ -121,6 +121,9 @@ public RESULT visit(CONTEXT ctx, Stack stack, Expr expr) throws PlanningEx case CreateIndex: current = visitCreateIndex(ctx, stack, (CreateIndex) expr); break; + case DropIndex: + current = visitDropIndex(ctx, stack, (DropIndex) expr); + break; case TruncateTable: current = visitTruncateTable(ctx, stack, (TruncateTable)expr); break; @@ -478,6 +481,11 @@ public RESULT visitCreateIndex(CONTEXT ctx, Stack stack, CreateIndex expr) return null; } + @Override + public RESULT visitDropIndex(CONTEXT ctx, Stack stack, DropIndex expr) throws PlanningException { + return null; + } + @Override public RESULT visitTruncateTable(CONTEXT ctx, Stack stack, TruncateTable expr) throws PlanningException { return null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java index 59422f1aeb..75cb3afd0a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java @@ -132,6 +132,9 @@ public RESULT visit(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock bl case CREATE_INDEX: current = visitCreateIndex(context, plan, block, (CreateIndexNode) node, stack); break; + case DROP_INDEX: + current = visitDropIndex(context, plan, block, (DropIndexNode) node, stack); + break; case TRUNCATE_TABLE: current = visitTruncateTable(context, plan, block, (TruncateTableNode) node, stack); break; @@ -357,6 +360,12 @@ public RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, QueryBlock blo return result; } + @Override + public RESULT visitDropIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, DropIndexNode node, + Stack stack) throws PlanningException { + return null; + } + @Override public RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node, Stack stack) throws PlanningException { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java index 9438d3452d..e19f0a1808 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java @@ -443,6 +443,13 @@ public LogicalNode visitCreateIndex(PreprocessContext ctx, Stack stack, Cr return createIndex; } + @Override + public LogicalNode visitDropIndex(PreprocessContext ctx, Stack stack, DropIndex expr) + throws PlanningException { + DropIndexNode dropIndexNode = ctx.plan.createNode(DropIndexNode.class); + return dropIndexNode; + } + public LogicalNode visitTruncateTable(PreprocessContext ctx, Stack stack, TruncateTable expr) throws PlanningException { TruncateTableNode truncateTableNode = ctx.plan.createNode(TruncateTableNode.class); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java index 57f943ef6c..7369ff7b6d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java @@ -97,6 +97,9 @@ RESULT visitAlterTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock RESULT visitCreateIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, CreateIndexNode node, Stack stack) throws PlanningException; + RESULT visitDropIndex(CONTEXT context, LogicalPlan plan, QueryBlock block, DropIndexNode node, + Stack stack) throws PlanningException; + RESULT visitTruncateTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, TruncateTableNode node, Stack stack) throws PlanningException; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index 58477b7f4a..846762ab7e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -45,8 +45,8 @@ import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.StorageUtil; -import org.apache.tajo.util.Pair; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import java.util.*; @@ -1873,10 +1873,18 @@ public LogicalNode visitCreateIndex(PlanContext context, Stack stack, Crea createIndexNode.setOptions(keyValueSet); } + createIndexNode.setInSchema(child.getOutSchema()); createIndexNode.setChild(child); return createIndexNode; } + @Override + public LogicalNode visitDropIndex(PlanContext context, Stack stack, DropIndex dropIndex) { + DropIndexNode dropIndexNode = context.queryBlock.getNodeFromExpr(dropIndex); + dropIndexNode.init(dropIndex.getIndexName(), dropIndex.isIfExists()); + return dropIndexNode; + } + @Override public LogicalNode visitTruncateTable(PlanContext context, Stack stack, TruncateTable truncateTable) throws PlanningException { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java index 55892ee29b..d80c945824 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java @@ -39,6 +39,18 @@ public class PlannerUtil { + public static boolean checkIfCreateIndexPlan(LogicalNode node) { + LogicalNode baseNode = node; + if (node instanceof LogicalRootNode) { + baseNode = ((LogicalRootNode) node).getChild(); + } + + NodeType type = baseNode.getType(); + + return + baseNode.getType() == NodeType.CREATE_INDEX; + } + public static boolean checkIfDDLPlan(LogicalNode node) { LogicalNode baseNode = node; if (node instanceof LogicalRootNode) { @@ -54,7 +66,9 @@ public static boolean checkIfDDLPlan(LogicalNode node) { baseNode.getType() == NodeType.DROP_TABLE || baseNode.getType() == NodeType.ALTER_TABLESPACE || baseNode.getType() == NodeType.ALTER_TABLE || - baseNode.getType() == NodeType.TRUNCATE_TABLE; + baseNode.getType() == NodeType.TRUNCATE_TABLE || + baseNode.getType() == NodeType.CREATE_INDEX || + baseNode.getType() == NodeType.DROP_INDEX; } /** diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java index 1dca608ce0..4c3395856c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateIndexNode.java @@ -29,10 +29,12 @@ public class CreateIndexNode extends UnaryNode implements Cloneable { @Expose private boolean isUnique; + @Expose private boolean isClustered; @Expose private String indexName; @Expose private SortSpec[] sortSpecs; @Expose private IndexMethod indexType = IndexMethod.TWO_LEVEL_BIN_TREE; @Expose private KeyValueSet options; + @Expose private boolean ifNotExists; public CreateIndexNode(int pid) { super(pid, NodeType.CREATE_INDEX); @@ -78,6 +80,14 @@ public KeyValueSet getOptions() { return this.options; } + public boolean isIfNotExists() { + return ifNotExists; + } + + public void setIfNotExists(boolean ifNotExists) { + this.ifNotExists = ifNotExists; + } + @Override public int hashCode() { return Objects.hashCode(isUnique, indexName, sortSpecs, indexType, options); @@ -130,4 +140,12 @@ public String toString() { public PlanString getPlanString() { return new PlanString(this); } + + public boolean isClustered() { + return isClustered; + } + + public void setClustered(boolean isClustered) { + this.isClustered = isClustered; + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropIndexNode.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropIndexNode.java new file mode 100644 index 0000000000..e50c0b960d --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropIndexNode.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.planner.logical; + +import com.google.common.base.Objects; +import com.google.gson.annotations.Expose; +import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.util.TUtil; + +public class DropIndexNode extends LogicalNode implements Cloneable { + @Expose private String indexName; + @Expose private boolean ifExists; + + public DropIndexNode(int pid) { + super(pid, NodeType.DROP_INDEX); + } + + public void init(String indexName, boolean ifExists) { + this.indexName = indexName; + this.ifExists = ifExists; + } + + @Override + public void preOrder(LogicalNodeVisitor visitor) { + visitor.visit(this); + } + + @Override + public void postOrder(LogicalNodeVisitor visitor) { + visitor.visit(this); + } + + @Override + public PlanString getPlanString() { + return new PlanString(this).appendTitle(ifExists ? " IF EXISTS" : ""); + } + + @Override + public String toString() { + return "DROP INDEX " + (ifExists ? "IF EXISTS " : "") + indexName; + } + + @Override + public int hashCode() { + return Objects.hashCode(indexName, ifExists); + } + + @Override + public boolean equals(Object o) { + if (o instanceof DropIndexNode) { + DropIndexNode other = (DropIndexNode) o; + return TUtil.checkEquals(this.indexName, other.indexName) && + this.ifExists == other.ifExists; + } + return false; + } + + @Override + public Object clone() throws CloneNotSupportedException { + DropIndexNode clone = (DropIndexNode) super.clone(); + clone.indexName = this.indexName; + clone.ifExists = this.ifExists; + return clone; + } + + public String getIndexName() { + return indexName; + } + + public boolean isIfExists() { + return ifExists; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java index 2e9e0dac5c..b8ba248be5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java @@ -56,6 +56,7 @@ public enum NodeType { ALTER_TABLESPACE (AlterTablespaceNode.class), ALTER_TABLE (AlterTableNode.class), CREATE_INDEX(CreateIndexNode.class), + DROP_INDEX(DropIndexNode.class), TRUNCATE_TABLE (TruncateTableNode.class); private final Class baseClass; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java index 91b5322331..5f1189ac3d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreIndexExec.java @@ -26,13 +26,14 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.logical.CreateIndexNode; import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.VTuple; +import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.storage.index.bst.BSTIndex; import org.apache.tajo.storage.index.bst.BSTIndex.BSTIndexWriter; import org.apache.tajo.worker.TaskAttemptContext; @@ -53,6 +54,14 @@ public StoreIndexExec(final TaskAttemptContext context, final CreateIndexNode lo this.logicalPlan = logicalPlan; } + private Path getIndexPath(TajoConf conf, String dbName, String indexName) { + // assume that there is only one fragment for each task + String table = context.getInputTables().iterator().next(); + FileFragment fragment = FragmentConvertor.convert(FileFragment.class, context.getTable(table)); + return new Path(TajoConf.getWarehouseDir(conf), dbName + "/" + indexName + "/" + + fragment.getStartKey() + "_" + fragment.getEndKey()); + } + @Override public void init() throws IOException { super.init(); @@ -70,9 +79,7 @@ public void init() throws IOException { TajoConf conf = context.getConf(); String[] splits = logicalPlan.getIndexName().split("\\."); - Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), splits[0] + "/" + splits[1] + "/" + - context.getUniqueKeyFromFragments()); - System.out.println("exec: " + indexPath); + Path indexPath = getIndexPath(conf, splits[0], splits[1]); // TODO: Create factory using reflection BSTIndex bst = new BSTIndex(conf); this.comparator = new TupleComparator(keySchema, sortSpecs); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 73f3cf5854..aed2fe6839 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -35,12 +35,14 @@ import org.apache.tajo.algebra.JsonHelper; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.IndexDesc.IndexKey; import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.exception.IllegalQueryStatusException; @@ -59,6 +61,7 @@ import org.apache.tajo.master.querymaster.QueryMasterTask; import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.*; +import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; @@ -187,10 +190,17 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, if (PlannerUtil.checkIfDDLPlan(rootNode)) { context.getSystemMetrics().counter("Query", "numDDLQuery").inc(); - updateQuery(session, rootNode.getChild()); - responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - responseBuilder.setResultCode(ClientProtos.ResultCode.OK); + if (PlannerUtil.checkIfCreateIndexPlan(rootNode)) { + return createIndex(session, (CreateIndexNode)rootNode.getChild(), queryContext, + plan, sql, jsonExpr, responseBuilder); + } else { + updateQuery(session, rootNode.getChild()); + responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); + responseBuilder.setResultCode(ClientProtos.ResultCode.OK); + + return responseBuilder.build(); + } } else if (plan.isExplain()) { // explain query String explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot()); Schema schema = new Schema(); @@ -216,8 +226,10 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, responseBuilder.setResultCode(ClientProtos.ResultCode.OK); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - // Simple query indicates a form of 'select * from tb_name [LIMIT X];'. + return responseBuilder.build(); + } else if (PlannerUtil.checkIfSimpleQuery(plan)) { + // Simple query indicates a form of 'select * from tb_name [LIMIT X];'. ScanNode scanNode = plan.getRootBlock().getNode(NodeType.SCAN); TableDesc desc = scanNode.getTableDesc(); if (plan.getRootBlock().hasNode(NodeType.LIMIT)) { @@ -232,8 +244,10 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); responseBuilder.setResultCode(ClientProtos.ResultCode.OK); - // NonFromQuery indicates a form of 'select a, x+y;' + return responseBuilder.build(); + } else if (PlannerUtil.checkIfNonFromQuery(plan)) { + // NonFromQuery indicates a form of 'select a, x+y;' Target [] targets = plan.getRootBlock().getRawTargets(); if (targets == null) { throw new PlanningException("No targets"); @@ -261,32 +275,40 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); responseBuilder.setResultCode(ClientProtos.ResultCode.OK); } + return responseBuilder.build(); + } else { // it requires distributed execution. So, the query is forwarded to a query master. context.getSystemMetrics().counter("Query", "numDMLQuery").inc(); - hookManager.doHooks(queryContext, plan); + return executeInCluster(queryContext, plan, session, sql, jsonExpr, responseBuilder); + } + } - QueryJobManager queryJobManager = this.context.getQueryJobManager(); - QueryInfo queryInfo; + private SubmitQueryResponse executeInCluster(QueryContext queryContext, LogicalPlan plan, Session session, String sql, String jsonExpr, + SubmitQueryResponse.Builder responseBuilder) + throws Exception { + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + hookManager.doHooks(queryContext, plan); - queryInfo = queryJobManager.createNewQueryJob(session, queryContext, sql, jsonExpr, rootNode); + QueryJobManager queryJobManager = this.context.getQueryJobManager(); + QueryInfo queryInfo; - if(queryInfo == null) { - responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); - responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR); - responseBuilder.setErrorMessage("Fail starting QueryMaster."); - } else { - responseBuilder.setIsForwarded(true); - responseBuilder.setQueryId(queryInfo.getQueryId().getProto()); - responseBuilder.setResultCode(ClientProtos.ResultCode.OK); - if(queryInfo.getQueryMasterHost() != null) { - responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost()); - } - responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort()); - LOG.info("Query is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort()); + queryInfo = queryJobManager.createNewQueryJob(session, queryContext, sql, jsonExpr, rootNode); + + if(queryInfo == null) { + responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); + responseBuilder.setResultCode(ClientProtos.ResultCode.ERROR); + responseBuilder.setErrorMessage("Fail starting QueryMaster."); + } else { + responseBuilder.setIsForwarded(true); + responseBuilder.setQueryId(queryInfo.getQueryId().getProto()); + responseBuilder.setResultCode(ClientProtos.ResultCode.OK); + if(queryInfo.getQueryMasterHost() != null) { + responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost()); } + responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort()); + LOG.info("Query is forwarded to " + queryInfo.getQueryMasterHost() + ":" + queryInfo.getQueryMasterPort()); } - SubmitQueryResponse response = responseBuilder.build(); - return response; + return responseBuilder.build(); } private void insertNonFromQuery(QueryContext queryContext, InsertNode insertNode, SubmitQueryResponse.Builder responseBuilder) @@ -448,6 +470,11 @@ private boolean updateQuery(Session session, LogicalNode root) throws IOExceptio TruncateTableNode truncateTable = (TruncateTableNode) root; truncateTable(session, truncateTable); return true; + case DROP_INDEX: + DropIndexNode dropIndexNode = (DropIndexNode) root; + dropIndex(session, dropIndexNode); + return true; + default: throw new InternalError("updateQuery cannot handle such query: \n" + root.toJson()); } @@ -636,6 +663,98 @@ public void truncateTable(final Session session, final TruncateTableNode truncat } } + /** + * Create an index for a given table. + * @param session user session + * @param createIndexNode the root of logical plan + */ + private SubmitQueryResponse createIndex(final Session session, final CreateIndexNode createIndexNode, + QueryContext queryContext, LogicalPlan plan, + String sql, String jsonExpr, + SubmitQueryResponse.Builder responseBuilder) throws Exception { + SubmitQueryResponse response = null; + final CatalogService catalog = context.getCatalog(); + final String dbName = session.getCurrentDatabase(); + String indexName = createIndexNode.getIndexName(); + if (CatalogUtil.isFQTableName(indexName)) { + indexName = CatalogUtil.splitFQTableName(indexName)[1]; + } + + boolean exists = catalog.existIndexByName(dbName, indexName); + if (exists) { + if (createIndexNode.isIfNotExists()) { + LOG.info("index \"" + indexName + "\" already exists." ); + } else { + throw new AlreadyExistsIndexException(createIndexNode.getIndexName()); + } + } else { + response = executeInCluster(queryContext, plan, session, sql, jsonExpr, responseBuilder); + + // get the table name and predicate from scan + try { + ScanNode scanNode = PlannerUtil.findTopNode(createIndexNode, NodeType.SCAN); + String tableName; + if (CatalogUtil.isFQTableName(scanNode.getTableName())) { + tableName = CatalogUtil.splitFQTableName(scanNode.getTableName())[1]; + } else { + tableName = scanNode.getTableName(); + } + String predicate = scanNode.hasQual() ? scanNode.getQual().toJson() : null; + // extract index keys + List indexKeys = TUtil.newList(); + for (SortSpec eachKey : createIndexNode.getSortSpecs()) { + indexKeys.add(new IndexKey(eachKey.getSortKey().toJson(), eachKey.isAscending(), eachKey.isNullFirst())); + } + + IndexDesc indexDesc = new IndexDesc(indexName, dbName, tableName, createIndexNode.getIndexType(), + indexKeys, createIndexNode.isUnique(), createIndexNode.isClustered(), predicate); + catalog.createIndex(indexDesc); + } catch (Exception e) { + // delete index + deleteIndexFiles(dbName, indexName); + } + } + return response; + } + + /** + * Drop the specified index. + * @param session user session + * @param indexName index name + * @param ifExists if exists + */ + public void dropIndex(final Session session, String indexName, boolean ifExists) { + final CatalogService catalog = context.getCatalog(); + final String dbName = session.getCurrentDatabase(); + + boolean exists = catalog.existIndexByName(dbName, indexName); + LOG.info("index name exist: " + exists); + if (!exists) { + if (ifExists) { + LOG.info("index \"" + indexName + "\" does not exist." ); + } else { + throw new NoSuchIndexException(indexName); + } + } else { + catalog.dropIndex(dbName, indexName); + deleteIndexFiles(dbName, indexName); + } + } + + private void deleteIndexFiles(String dbName, String indexName) { + Path indexPath = new Path(context.getConf().getVar(ConfVars.WAREHOUSE_DIR), dbName + "/" + indexName); + try { + FileSystem fs = indexPath.getFileSystem(context.getConf()); + fs.delete(indexPath, true); + } catch (IOException e) { + throw new InternalError(e.getMessage()); + } + } + + private void dropIndex(final Session session, final DropIndexNode dropIndexNode) { + dropIndex(session, dropIndexNode.getIndexName(), dropIndexNode.isIfExists()); + } + private boolean existColumnName(String tableName, String columnName) { final TableDesc tableDesc = catalog.getTableDesc(tableName); return tableDesc.getSchema().containsByName(columnName) ? true : false; @@ -692,7 +811,7 @@ public TableDesc createTableOnPath(Session session, String tableName, Schema sch if (exists) { if (ifNotExists) { - LOG.info("relation \"" + qualifiedName + "\" is already exists." ); + LOG.info("relation \"" + qualifiedName + "\" already exists." ); return catalog.getTableDesc(databaseName, simpleTableName); } else { throw new AlreadyExistsTableException(CatalogUtil.buildFQName(databaseName, tableName)); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 97f59ef734..1ac85e382d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -27,7 +27,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; -import org.apache.tajo.*; +import org.apache.tajo.QueryId; +import org.apache.tajo.QueryIdFactory; +import org.apache.tajo.TajoIdProtos; +import org.apache.tajo.TajoProtos; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.NoSuchDatabaseException; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -751,5 +754,42 @@ public FunctionResponse getFunctionList(RpcController controller, SessionedStrin throw new ServiceException(t); } } + + @Override + public BoolProto existIndex(RpcController controller, SessionedStringProto request) throws ServiceException { + try { + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + + String databaseName; + String indexName; + if (CatalogUtil.isFQTableName(request.getValue())) { + String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + databaseName = splitted[0]; + indexName = splitted[1]; + } else { + databaseName = session.getCurrentDatabase(); + indexName = request.getValue(); + } + + if (catalog.existIndexByName(databaseName, indexName)) { + return BOOL_TRUE; + } else { + return BOOL_FALSE; + } + } catch (Throwable e) { + throw new ServiceException(e); + } + } + + @Override + public BoolProto dropIndex(RpcController controller, SessionedStringProto request) throws ServiceException { + try { + Session session = context.getSessionManager().getSession(request.getSessionId().getId()); + context.getGlobalEngine().dropIndex(session, request.getValue(), true); + return BOOL_TRUE; + } catch (Throwable t) { + throw new ServiceException(t); + } + } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index 7ecf27af67..2b67538420 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -294,14 +294,6 @@ public Collection getInputTables() { return fragmentMap.get(id).toArray(new FragmentProto[fragmentMap.get(id).size()]); } - public long getUniqueKeyFromFragments() { - List totalFragments = new ArrayList(); - for (List eachFragments : fragmentMap.values()) { - totalFragments.addAll(eachFragments); - } - return Objects.hashCode(totalFragments.toArray(new FragmentProto[totalFragments.size()])); - } - public int hashCode() { return Objects.hashCode(queryId); } diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index b97eb3070b..7e87c292b3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -36,7 +36,9 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.FileUtil; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.rules.TestName; import java.io.File; @@ -46,7 +48,10 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import static org.junit.Assert.*; @@ -381,6 +386,16 @@ public void assertTableExists(String tableName) throws ServiceException { assertTrue(client.existTable(tableName)); } + /** + * Assert that the index exists. + * + * @param indexName The index name to be checked. This name is case sensitive. + * @throws ServiceException + */ + public void assertIndexExists(String indexName) throws ServiceException { + assertTrue(client.existIndex(indexName)); + } + /** * Assert that the table does not exist. * @@ -390,6 +405,10 @@ public void assertTableNotExists(String tableName) throws ServiceException { assertTrue(!client.existTable(tableName)); } + public void assertIndexNotExists(String indexName) throws ServiceException { + assertFalse(client.existIndex(indexName)); + } + public void assertColumnExists(String tableName,String columnName) throws ServiceException { TableDesc tableDesc = fetchTableMetaData(tableName); assertTrue(tableDesc.getSchema().containsByName(columnName)); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 2ca85cc545..6a6d07cb2c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -54,7 +54,6 @@ import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; public class TestLogicalPlanner { private static TajoTestingCluster util; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 920c9329b5..6c36040ba3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -34,7 +34,6 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; @@ -798,7 +797,7 @@ public final void testCreateIndex() throws IOException, PlanningException { FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex"); - Path indexPath = StorageUtil.concatPath(conf.getVar(ConfVars.ROOT_DIR), "default/idx_employee"); + Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee"); if (sm.getFileSystem().exists(indexPath)) { sm.getFileSystem().delete(indexPath, true); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java index 2a66909640..4714bd6a4b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateIndex.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.query; +import com.google.protobuf.ServiceException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; @@ -29,7 +30,9 @@ import java.io.IOException; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; @Category(IntegrationTest.class) public class TestCreateIndex extends QueryTestCaseBase { @@ -38,35 +41,50 @@ public TestCreateIndex() { super(TajoConstants.DEFAULT_DATABASE_NAME); } - private static void assertIndexExist(String indexName) throws IOException { + private void checkIndexExist(String indexName) throws IOException, ServiceException { Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName); FileSystem fs = indexPath.getFileSystem(conf); assertTrue(fs.exists(indexPath)); assertEquals(2, fs.listStatus(indexPath).length); - fs.deleteOnExit(indexPath); + assertIndexExists(indexName); + } + + private void checkIndexNotExist(String indexName) throws IOException, ServiceException { + Path indexPath = new Path(conf.getVar(ConfVars.WAREHOUSE_DIR), "default/" + indexName); + FileSystem fs = indexPath.getFileSystem(conf); + assertFalse(fs.exists(indexPath)); + assertIndexNotExists(indexName); } @Test public final void testCreateIndex() throws Exception { executeQuery(); - assertIndexExist("l_orderkey_idx"); + checkIndexExist("l_orderkey_idx"); + executeString("drop index l_orderkey_idx"); + checkIndexNotExist("l_orderkey_idx"); } @Test public final void testCreateIndexOnMultiAttrs() throws Exception { executeQuery(); - assertIndexExist("l_orderkey_partkey_idx"); + checkIndexExist("l_orderkey_partkey_idx"); + executeString("drop index l_orderkey_partkey_idx"); + checkIndexNotExist("l_orderkey_partkey_idx"); } @Test public final void testCreateIndexWithCondition() throws Exception { executeQuery(); - assertIndexExist("l_orderkey_partkey_lt10_idx"); + checkIndexExist("l_orderkey_partkey_lt10_idx"); + executeString("drop index l_orderkey_partkey_lt10_idx"); + checkIndexNotExist("l_orderkey_partkey_lt10_idx"); } @Test public final void testCreateIndexOnExpression() throws Exception { executeQuery(); - assertIndexExist("l_orderkey_100_lt10_idx"); + checkIndexExist("l_orderkey_100_lt10_idx"); + executeString("drop index l_orderkey_100_lt10_idx"); + checkIndexNotExist("l_orderkey_100_lt10_idx"); } } From ca5fca5b784be4d6afcaca7e72ff3821eb41b931 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 21 Jul 2014 12:41:43 +0900 Subject: [PATCH 2/2] TAJO-837 --- .../tajo/catalog/store/AbstractDBStore.java | 15 --------------- .../apache/tajo/catalog/store/CatalogStore.java | 6 ------ 2 files changed, 21 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 83811bcdfd..449ec745fd 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -679,21 +679,7 @@ public void createTable(final CatalogProtos.TableDescProto table) throws Catalog pstmt.close(); } -// String tidSql = -// "SELECT TID from " + TB_TABLES + " WHERE " + COL_DATABASES_PK + "=? AND " + COL_TABLES_NAME + "=?"; -// pstmt = conn.prepareStatement(tidSql); -// pstmt.setInt(1, dbid); -// pstmt.setString(2, tableName); -// res = pstmt.executeQuery(); -// -// if (!res.next()) { -// throw new CatalogException("ERROR: there is no TID matched to " + table.getTableName()); -// } -// -// int tableId = res.getInt("TID"); int tableId = getTableId(dbid, databaseName, tableName); -// res.close(); -// pstmt.close(); String colSql = "INSERT INTO " + TB_COLUMNS + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?) "; @@ -1841,7 +1827,6 @@ public IndexDescProto[] getIndexes(String databaseName, final String tableName) private void resultToIndexDescProtoBuilder(IndexDescProto.Builder builder, final ResultSet res) throws SQLException { builder.setIndexName(res.getString("index_name")); -// builder.setkeys(indexResultToColumnProto(res)); builder.setMethod(getIndexMethod(res.getString("method").trim())); builder.setIsUnique(res.getBoolean("is_unique")); builder.setIsClustered(res.getBoolean("is_clustered")); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 46fab64512..ce4ea79f97 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -105,14 +105,8 @@ void addPartition(String databaseName, String tableName, IndexDescProto getIndexByName(String databaseName, String indexName) throws CatalogException; -// IndexDescProto getIndexByColumn(String databaseName, String tableName, String columnName) -// throws CatalogException; - boolean existIndexByName(String databaseName, String indexName) throws CatalogException; -// boolean existIndexByColumn(String databaseName, String tableName, String columnName) -// throws CatalogException; - IndexDescProto [] getIndexes(String databaseName, String tableName) throws CatalogException; /************************** FUNCTION *****************************/