From 7572f6d71ba2dff9010e662b512068b6b4027c87 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Sat, 25 Jul 2015 13:11:23 +0900 Subject: [PATCH 01/25] TAJO-1675: NPE when selecting data from information_schema.partition_keys. --- .../tajo/catalog/AbstractCatalogClient.java | 14 +++++++++ .../src/main/proto/CatalogProtocol.proto | 6 ++++ .../apache/tajo/catalog/CatalogService.java | 3 ++ .../src/main/proto/CatalogProtos.proto | 6 ++++ .../tajo/catalog/store/HiveCatalogStore.java | 6 ++++ .../apache/tajo/catalog/CatalogServer.java | 23 ++++++++++++++ .../tajo/catalog/store/AbstractDBStore.java | 31 +++++++++++++++++++ .../tajo/catalog/store/CatalogStore.java | 5 +-- .../apache/tajo/catalog/store/MemStore.java | 24 ++++++++++++++ .../NonForwardQueryResultSystemScanner.java | 31 ++++++++++++++++++- .../tajo/engine/query/TestAlterTable.java | 27 ++++++++++++++++ 11 files changed, 173 insertions(+), 3 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index f7f77851dc..148c0adaac 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -401,6 +401,20 @@ public List getAllPartitions() { } } + @Override + public List getAllPartitionKeys() { + try { + final BlockingInterface stub = getStub(); + final GetTablePartitionKeysResponse response = stub.getAllPartitionKeys(null, ProtoUtil.NULL_PROTO); + ensureOk(response.getState()); + + return response.getPartKeyList(); + + } catch (ServiceException e) { + throw new RuntimeException(e); + } + } + @Override public final Collection getAllTableNames(final String databaseName) { try { diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index 8d0eef659b..d5d5d8b3d8 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -89,6 +89,11 @@ message GetTablePartitionsResponse { repeated TablePartitionProto part = 2; } +message GetTablePartitionKeysResponse { + required ReturnState state = 1; + repeated TablePartitionKeysProto partKey = 2; +} + service CatalogProtocolService { rpc createTablespace(CreateTablespaceRequest) returns (ReturnState); @@ -124,6 +129,7 @@ service CatalogProtocolService { rpc getPartitionByPartitionName(PartitionIdentifierProto) returns (GetPartitionDescResponse); rpc getPartitionsByTableName(PartitionIdentifierProto) returns (GetPartitionsResponse); rpc getAllPartitions(NullProto) returns (GetTablePartitionsResponse); + rpc getAllPartitionKeys(NullProto) returns (GetTablePartitionKeysResponse); rpc createIndex(IndexDescProto) returns (ReturnState); rpc dropIndex(IndexNameProto) returns (ReturnState); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index 5dc54126e1..02dba0b8d8 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -28,6 +28,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionKeysProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; import org.apache.tajo.common.TajoDataTypes.DataType; @@ -195,6 +196,8 @@ CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableN List getAllPartitions(); + List getAllPartitionKeys(); + boolean createIndex(IndexDesc index); boolean existIndexByName(String databaseName, String indexName); diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index f95df0a40a..f1e8237cce 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -161,6 +161,12 @@ message TablePartitionProto { optional string path = 4; } +message TablePartitionKeysProto { + required int32 partition_id = 1; + required string columnName = 2; + required string partitionValue = 3; +} + message GetIndexByColumnRequest { required TableIdentifierProto tableIdentifier = 1; required string columnName = 2; diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 8a08b77a3e..079979bcce 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -45,6 +45,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionKeysProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; import org.apache.tajo.catalog.proto.CatalogProtos.TablespaceProto; import org.apache.tajo.catalog.statistics.TableStats; @@ -940,6 +941,11 @@ public List getAllPartitions() throws CatalogException { throw new UnsupportedOperationException(); } + @Override + public List getAllPartitionKeys() throws CatalogException { + throw new UnsupportedOperationException(); + } + @Override public List getAllTableProperties() throws CatalogException { throw new UnsupportedOperationException(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 6a3fc166d8..fd4c515ba9 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -1062,6 +1062,29 @@ public GetTablePartitionsResponse getAllPartitions(RpcController controller, Nul } } + @Override + public GetTablePartitionKeysResponse getAllPartitionKeys(RpcController controller, + NullProto request) throws ServiceException { + rlock.lock(); + + try { + return GetTablePartitionKeysResponse.newBuilder() + .setState(OK) + .addAllPartKey(store.getAllPartitionKeys()) + .build(); + + } catch (Throwable t) { + printStackTraceIfError(LOG, t); + + return GetTablePartitionKeysResponse.newBuilder() + .setState(returnError(t)) + .build(); + + } finally { + rlock.unlock(); + } + } + @Override public ReturnState createIndex(RpcController controller, IndexDescProto indexDesc) { String dbName = indexDesc.getTableIdentifier().getDatabaseName(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index c6b7d36315..7a4884d174 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2171,6 +2171,37 @@ public List getAllPartitions() throws CatalogException { return partitions; } + @Override + public List getAllPartitionKeys() throws CatalogException { + Connection conn = null; + Statement stmt = null; + ResultSet resultSet = null; + + List partitions = new ArrayList(); + + try { + String sql = "SELECT " + COL_PARTITIONS_PK + ", COLUMN_NAME, PARTITION_VALUE FROM " + TB_PARTTION_KEYS; + + conn = getConnection(); + stmt = conn.createStatement(); + resultSet = stmt.executeQuery(sql); + while (resultSet.next()) { + TablePartitionKeysProto.Builder builder = TablePartitionKeysProto.newBuilder(); + + builder.setPartitionId(resultSet.getInt(COL_PARTITIONS_PK)); + builder.setColumnName(resultSet.getString("COLUMN_NAME")); + builder.setPartitionValue(resultSet.getString("PARTITION_VALUE")); + + partitions.add(builder.build()); + } + } catch (SQLException se) { + throw new TajoInternalError(se); + } finally { + CatalogUtil.closeQuietly(stmt, resultSet); + } + + return partitions; + } @Override public void createIndex(final IndexDescProto proto) throws CatalogException { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 4ffedcfc62..76e4720a41 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -27,13 +27,12 @@ import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionKeysProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; import java.io.Closeable; import org.apache.tajo.catalog.exception.CatalogException; -import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; -import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto; import java.util.Collection; import java.util.List; @@ -117,6 +116,8 @@ CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableN List getAllPartitions() throws CatalogException; + List getAllPartitionKeys() throws CatalogException; + /**************************** INDEX *******************************/ void createIndex(IndexDescProto proto) throws CatalogException; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 5763f31f58..8f3dfb5e8a 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -39,6 +39,8 @@ import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto; import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionProto; +import org.apache.tajo.catalog.proto.CatalogProtos.TablePartitionKeysProto; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto; import org.apache.tajo.util.KeyValueSet; @@ -603,6 +605,28 @@ public List getAllPartitions() throws CatalogException { return protos; } + public List getAllPartitionKeys() throws CatalogException { + List protos = new ArrayList(); + Set tables = partitions.keySet(); + int partitionId = 0; + for (String table : tables) { + Map entryMap = partitions.get(table); + for (Map.Entry proto : entryMap.entrySet()) { + CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); + + for (PartitionKeyProto partitionKey : partitionDescProto.getPartitionKeysList()) { + TablePartitionKeysProto.Builder builder = TablePartitionKeysProto.newBuilder(); + builder.setColumnName(partitionKey.getColumnName()); + builder.setPartitionValue(partitionKey.getPartitionValue()); + builder.setPartitionId(partitionId); + protos.add(builder.build()); + } + partitionId++; + } + } + return protos; + } + /* (non-Javadoc) * @see CatalogStore#createIndex(nta.catalog.proto.CatalogProtos.IndexDescProto) */ diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 228c0b8fb0..cdf792bd4c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -422,7 +422,34 @@ private List getAllPartitions(Schema outSchema) { return tuples; } - + + private List getAllPartitionKeys(Schema outSchema) { + List partitionKeyList = masterContext.getCatalog().getAllPartitionKeys(); + List tuples = new ArrayList(partitionKeyList.size()); + List columns = outSchema.getRootColumns(); + Tuple aTuple; + + for (TablePartitionKeysProto partitionKey: partitionKeyList) { + aTuple = new VTuple(outSchema.size()); + + for (int fieldId = 0; fieldId < columns.size(); fieldId++) { + Column column = columns.get(fieldId); + + if ("partition_id".equalsIgnoreCase(column.getSimpleName())) { + aTuple.put(fieldId, DatumFactory.createInt4(partitionKey.getPartitionId())); + } else if ("column_name".equalsIgnoreCase(column.getSimpleName())) { + aTuple.put(fieldId, DatumFactory.createText(partitionKey.getColumnName())); + } else if ("partition_value".equalsIgnoreCase(column.getSimpleName())) { + aTuple.put(fieldId, DatumFactory.createText(partitionKey.getPartitionValue())); + } + } + + tuples.add(aTuple); + } + + return tuples; + } + private Tuple getQueryMasterTuple(Schema outSchema, NodeStatus aNodeStatus) { List columns = outSchema.getRootColumns(); Tuple aTuple = new VTuple(outSchema.size()); @@ -581,6 +608,8 @@ private List fetchSystemTable(TableDesc tableDesc, Schema inSchema) { tuples = getAllTableStats(inSchema); } else if ("partitions".equalsIgnoreCase(tableName)) { tuples = getAllPartitions(inSchema); + } else if ("partition_keys".equalsIgnoreCase(tableName)) { + tuples = getAllPartitionKeys(inSchema); } else if ("cluster".equalsIgnoreCase(tableName)) { tuples = getClusterInfo(inSchema); } else if ("session".equalsIgnoreCase(tableName)) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 8cdaf80d8a..b50e8d5477 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -28,6 +28,7 @@ import org.junit.Test; import org.junit.experimental.categories.Category; +import javax.xml.transform.Result; import java.sql.ResultSet; import java.util.List; @@ -103,6 +104,32 @@ public final void testAlterTableAddPartition() throws Exception { assertTrue(fs.exists(partitionPath)); assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); + List tablePartitions = catalog.getAllPartitions(); + assertEquals(tablePartitions.size(), 1); + assertEquals(tablePartitions.get(0).getPartitionName(), "col3=1/col4=2"); + + List tablePartitionKeys = catalog.getAllPartitionKeys(); + assertEquals(tablePartitionKeys.size(), 2); + assertEquals(tablePartitionKeys.get(0).getColumnName(), "col3"); + assertEquals(tablePartitionKeys.get(0).getPartitionValue(), "1"); + assertEquals(tablePartitionKeys.get(1).getColumnName(), "col4"); + assertEquals(tablePartitionKeys.get(1).getPartitionValue(), "2"); + + ResultSet resultSet = executeString("SELECT partition_name FROM INFORMATION_SCHEMA.PARTITIONS"); + String actualResult = resultSetToString(resultSet); + String expectedResult = "partition_name\n" + + "-------------------------------\n" + + "col3=1/col4=2\n"; + assertEquals(expectedResult, actualResult); + + resultSet = executeString("SELECT * FROM INFORMATION_SCHEMA.PARTITION_KEYS"); + actualResult = resultSetToString(resultSet); + expectedResult = "partition_id,column_name,partition_value\n" + + "-------------------------------\n" + + "0,col3,1\n" + + "0,col4,2\n"; + assertEquals(expectedResult, actualResult); + executeDDL("alter_table_drop_partition1.sql", null); partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); From f83ae11c093ab0a74902fe99973114f0d1c921a7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 27 Jul 2015 14:46:05 +0900 Subject: [PATCH 02/25] Add debug log temporarily --- .../apache/tajo/engine/query/TestAlterTable.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index b50e8d5477..7b489d0587 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -18,6 +18,8 @@ package org.apache.tajo.engine.query; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; @@ -37,6 +39,8 @@ @Category(IntegrationTest.class) public class TestAlterTable extends QueryTestCaseBase { + private static final Log logger = LogFactory.getLog(TestAlterTable.class); + @Test public final void testAlterTableName() throws Exception { List createdNames = executeDDL("table1_ddl.sql", "table1.tbl", "ABC"); @@ -105,10 +109,22 @@ public final void testAlterTableAddPartition() throws Exception { assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); List tablePartitions = catalog.getAllPartitions(); + /// Debug Log + for(CatalogProtos.TablePartitionProto tablePartition : tablePartitions) { + logger.info("### tablePartition:" + tablePartition.getPartitionName() + + ", path:" + tablePartition.getPath()); + } + assertEquals(tablePartitions.size(), 1); assertEquals(tablePartitions.get(0).getPartitionName(), "col3=1/col4=2"); List tablePartitionKeys = catalog.getAllPartitionKeys(); + /// Debug Log + for(CatalogProtos.TablePartitionKeysProto tablePartitionKey : tablePartitionKeys) { + logger.info("### tablePartitionKey:" + tablePartitionKey.getColumnName() + + ", value:" + tablePartitionKey.getPartitionValue()); + } + assertEquals(tablePartitionKeys.size(), 2); assertEquals(tablePartitionKeys.get(0).getColumnName(), "col3"); assertEquals(tablePartitionKeys.get(0).getPartitionValue(), "1"); From 9e8e6ef007bd38f413d5f939be9d40debad28727 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 27 Jul 2015 15:10:45 +0900 Subject: [PATCH 03/25] Update debug log --- .../org/apache/tajo/engine/query/TestAlterTable.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 7b489d0587..0def5429ff 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -18,8 +18,6 @@ package org.apache.tajo.engine.query; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; @@ -30,16 +28,13 @@ import org.junit.Test; import org.junit.experimental.categories.Category; -import javax.xml.transform.Result; import java.sql.ResultSet; import java.util.List; -import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.*; @Category(IntegrationTest.class) public class TestAlterTable extends QueryTestCaseBase { - private static final Log logger = LogFactory.getLog(TestAlterTable.class); @Test public final void testAlterTableName() throws Exception { @@ -110,8 +105,9 @@ public final void testAlterTableAddPartition() throws Exception { List tablePartitions = catalog.getAllPartitions(); /// Debug Log + System.out.println("### tablePartitions - size:" + tablePartitions.size()); for(CatalogProtos.TablePartitionProto tablePartition : tablePartitions) { - logger.info("### tablePartition:" + tablePartition.getPartitionName() + System.out.println("### tablePartition:" + tablePartition.getPartitionName() + ", path:" + tablePartition.getPath()); } @@ -120,8 +116,9 @@ public final void testAlterTableAddPartition() throws Exception { List tablePartitionKeys = catalog.getAllPartitionKeys(); /// Debug Log + System.out.println("### tablePartitionKeys - size:" + tablePartitionKeys.size()); for(CatalogProtos.TablePartitionKeysProto tablePartitionKey : tablePartitionKeys) { - logger.info("### tablePartitionKey:" + tablePartitionKey.getColumnName() + System.out.println("### tablePartitionKey:" + tablePartitionKey.getColumnName() + ", value:" + tablePartitionKey.getPartitionValue()); } From 0587ac544fb1d18be5c1ecc2213c51611d5bac12 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 31 Jul 2015 15:32:52 +0900 Subject: [PATCH 04/25] Fix unit test error --- .../apache/tajo/catalog/store/MemStore.java | 24 ++++++++------ .../tajo/engine/query/TestAlterTable.java | 32 +++++++++++++++++-- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index b083f42f4c..a26d0f8f09 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -586,10 +586,19 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String } public List getAllPartitions() throws CatalogException { + int tableId = 0, i = 0; + List tables = getAllTables(); List protos = new ArrayList(); - Set tables = partitions.keySet(); - for (String table : tables) { - Map entryMap = partitions.get(table); + + Set partitionTables = partitions.keySet(); + for (String partitionTable : partitionTables) { + for (TableDescriptorProto table : tables) { + if (table.getName().equals(partitionTable)) { + tableId = table.getTid(); + } + } + + Map entryMap = partitions.get(partitionTable); for (Map.Entry proto : entryMap.entrySet()) { CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); @@ -597,14 +606,11 @@ public List getAllPartitions() throws CatalogException { builder.setPartitionName(partitionDescProto.getPartitionName()); builder.setPath(partitionDescProto.getPath()); - - // PARTITION_ID and TID is always necessary variables. In other CatalogStore excepting MemStore, - // all partitions would have PARTITION_ID and TID. But MemStore doesn't contain these variable values because - // it is implemented for test purpose. Thus, we need to set each variables to 0. - builder.setPartitionId(0); - builder.setTid(0); + builder.setPartitionId(i); + builder.setTid(tableId); protos.add(builder.build()); + i++; } } return protos; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 35dc0435a8..45c42b5e26 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -25,6 +25,7 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.util.TUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -103,9 +104,34 @@ public final void testAlterTableAddPartition() throws Exception { assertTrue(fs.exists(partitionPath)); assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); - List tablePartitions = catalog.getAllPartitions(); - assertEquals(tablePartitions.size(), 1); - assertEquals(tablePartitions.get(0).getPartitionName(), "col3=1/col4=2"); + List allDatabases = catalog.getAllDatabases(); + int dbId = -1; + for (CatalogProtos.DatabaseProto database : allDatabases) { + if (database.getName().equals("TestAlterTable")) { + dbId = database.getId(); + } + } + assertNotEquals(dbId, -1); + + int tableId = -1; + List allTables = catalog.getAllTables(); + for(CatalogProtos.TableDescriptorProto table : allTables) { + if (table.getDbId() == dbId && table.getName().equals("partitioned_table")) { + tableId = table.getTid(); + } + } + assertNotEquals(tableId, -1); + + List allPartitions = catalog.getAllPartitions(); + List resultPartitions = TUtil.newList(); + + for (CatalogProtos.TablePartitionProto partition : allPartitions) { + if (partition.getTid() == tableId) { + resultPartitions.add(partition); + } + } + assertEquals(resultPartitions.size(), 1); + assertEquals(resultPartitions.get(0).getPartitionName(), "col3=1/col4=2"); List tablePartitionKeys = catalog.getAllPartitionKeys(); assertEquals(tablePartitionKeys.size(), 2); From d3dacd9a82836c00337f008455cb9d1a7c673964 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 4 Aug 2015 15:41:06 +0900 Subject: [PATCH 05/25] Fix unit test bug and add some debug logs --- .../apache/tajo/catalog/store/MemStore.java | 16 +++-- .../tajo/engine/query/TestAlterTable.java | 65 ++++++++++++++----- 2 files changed, 58 insertions(+), 23 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 2f6443e0e4..3029806159 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -354,7 +354,6 @@ private void addPartition(CatalogProtos.PartitionDescProto partitionDesc, String CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); builder.setPath(partitionDesc.getPath()); - if (partitionDesc.getPartitionKeysCount() > 0) { for (CatalogProtos.PartitionKeyProto eachKey : partitionDesc.getPartitionKeysList()) { CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder(); @@ -593,7 +592,7 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String } public List getAllPartitions() throws CatalogException { - int tableId = 0, i = 0; + int tableId = 0, partitionId = 0; List tables = getAllTables(); List protos = new ArrayList(); @@ -613,11 +612,11 @@ public List getAllPartitions() throws CatalogException { builder.setPartitionName(partitionDescProto.getPartitionName()); builder.setPath(partitionDescProto.getPath()); - builder.setPartitionId(i); + builder.setPartitionId(partitionId); builder.setTid(tableId); protos.add(builder.build()); - i++; + partitionId++; } } return protos; @@ -625,10 +624,11 @@ public List getAllPartitions() throws CatalogException { public List getAllPartitionKeys() throws CatalogException { List protos = new ArrayList(); - Set tables = partitions.keySet(); int partitionId = 0; - for (String table : tables) { - Map entryMap = partitions.get(table); + + Set partitionTables = partitions.keySet(); + for (String partitionTable : partitionTables) { + Map entryMap = partitions.get(partitionTable); for (Map.Entry proto : entryMap.entrySet()) { CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); @@ -639,9 +639,11 @@ public List getAllPartitionKeys() throws CatalogExcepti builder.setPartitionId(partitionId); protos.add(builder.build()); } + partitionId++; } } + return protos; } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index c414b7406a..9e7da2fce0 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -77,7 +77,8 @@ public final void testAlterTableSetProperty() throws Exception { public final void testAlterTableAddPartition() throws Exception { executeDDL("create_partitioned_table.sql", null); - String tableName = CatalogUtil.buildFQName("TestAlterTable", "partitioned_table"); + String simpleTableName = "partitioned_table"; + String tableName = CatalogUtil.buildFQName(getCurrentDatabase(), simpleTableName); assertTrue(catalog.existsTable(tableName)); TableDesc retrieved = catalog.getTableDesc(tableName); @@ -90,7 +91,8 @@ public final void testAlterTableAddPartition() throws Exception { executeDDL("alter_table_add_partition1.sql", null); executeDDL("alter_table_add_partition2.sql", null); - List partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + List partitions = catalog.getPartitions(getCurrentDatabase() + , simpleTableName); assertNotNull(partitions); assertEquals(partitions.size(), 1); assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2"); @@ -108,7 +110,7 @@ public final void testAlterTableAddPartition() throws Exception { List allDatabases = catalog.getAllDatabases(); int dbId = -1; for (CatalogProtos.DatabaseProto database : allDatabases) { - if (database.getName().equals("TestAlterTable")) { + if (database.getName().equals(getCurrentDatabase())) { dbId = database.getId(); } } @@ -117,7 +119,7 @@ public final void testAlterTableAddPartition() throws Exception { int tableId = -1; List allTables = catalog.getAllTables(); for(CatalogProtos.TableDescriptorProto table : allTables) { - if (table.getDbId() == dbId && table.getName().equals("partitioned_table")) { + if (table.getDbId() == dbId && table.getName().equals(simpleTableName)) { tableId = table.getTid(); } } @@ -126,40 +128,71 @@ public final void testAlterTableAddPartition() throws Exception { List allPartitions = catalog.getAllPartitions(); List resultPartitions = TUtil.newList(); + System.out.println("### tablePath:" + retrieved.getUri().toString()); + System.out.println("### tableId:" + tableId); + + int partitionId = 0; for (CatalogProtos.TablePartitionProto partition : allPartitions) { - if (partition.getTid() == tableId) { + System.out.println("### partition - partitionId:" + partition.getPartitionId() + + ", tid:" + partition.getTid() + + ", name:" + partition.getPartitionName() + + ", path:" + partition.getPath() + ); + + if (partition.getTid() == tableId + && partition.getPartitionName().equals("col3=1/col4=2") + && partition.getPath().equals(retrieved.getUri().toString() + "/col3=1/col4=2") + ){ resultPartitions.add(partition); + partitionId = partition.getPartitionId(); } } assertEquals(resultPartitions.size(), 1); assertEquals(resultPartitions.get(0).getPartitionName(), "col3=1/col4=2"); List tablePartitionKeys = catalog.getAllPartitionKeys(); - assertEquals(tablePartitionKeys.size(), 2); - assertEquals(tablePartitionKeys.get(0).getColumnName(), "col3"); - assertEquals(tablePartitionKeys.get(0).getPartitionValue(), "1"); - assertEquals(tablePartitionKeys.get(1).getColumnName(), "col4"); - assertEquals(tablePartitionKeys.get(1).getPartitionValue(), "2"); + List resultPartitionKeys = TUtil.newList(); + + for (CatalogProtos.TablePartitionKeysProto partitionKey: tablePartitionKeys) { + System.out.println("### partitionKeys - partitionId:" + partitionKey.getPartitionId() + + ", column:" + partitionKey.getColumnName() + + ", value:" + partitionKey.getPartitionValue() + ); + if (partitionKey.getPartitionId() == partitionId + && (partitionKey.getColumnName().equals("col3") && partitionKey.getPartitionValue().equals("1") + || partitionKey.getColumnName().equals("col4") && partitionKey.getPartitionValue().equals("2"))) { + resultPartitionKeys.add(partitionKey); + } + } + assertEquals(resultPartitionKeys.size(), 2); + assertEquals(resultPartitionKeys.get(0).getColumnName(), "col3"); + assertEquals(resultPartitionKeys.get(0).getPartitionValue(), "1"); + assertEquals(resultPartitionKeys.get(1).getColumnName(), "col4"); + assertEquals(resultPartitionKeys.get(1).getPartitionValue(), "2"); + + ResultSet resultSet = executeString("SELECT partition_name FROM INFORMATION_SCHEMA.PARTITIONS " + + " WHERE partition_id = " + partitionId); - ResultSet resultSet = executeString("SELECT partition_name FROM INFORMATION_SCHEMA.PARTITIONS"); String actualResult = resultSetToString(resultSet); String expectedResult = "partition_name\n" + "-------------------------------\n" + "col3=1/col4=2\n"; assertEquals(expectedResult, actualResult); - resultSet = executeString("SELECT * FROM INFORMATION_SCHEMA.PARTITION_KEYS"); + resultSet = executeString("SELECT column_name,partition_value FROM INFORMATION_SCHEMA.PARTITION_KEYS" + + " WHERE partition_id = " + partitionId); + actualResult = resultSetToString(resultSet); - expectedResult = "partition_id,column_name,partition_value\n" + + expectedResult = "column_name,partition_value\n" + "-------------------------------\n" + - "0,col3,1\n" + - "0,col4,2\n"; + "col3,1\n" + + "col4,2\n"; assertEquals(expectedResult, actualResult); executeDDL("alter_table_drop_partition1.sql", null); executeDDL("alter_table_drop_partition2.sql", null); - partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + partitions = catalog.getPartitions(getCurrentDatabase(), simpleTableName); assertNotNull(partitions); assertEquals(partitions.size(), 0); assertFalse(fs.exists(partitionPath)); From 651bfcddf22a2a00ff5392203ba719c2dbe7a666 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 4 Aug 2015 16:01:47 +0900 Subject: [PATCH 06/25] Optimize queries for getting partition informs --- .../apache/tajo/catalog/store/AbstractDBStore.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 6648d7e7ff..e4df8860d7 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2122,8 +2122,10 @@ public List getAllPartitions() throws CatalogException { List partitions = new ArrayList(); try { - String sql = "SELECT " + COL_PARTITIONS_PK + ", " + COL_TABLES_PK + ", PARTITION_NAME, " + - " PATH FROM " + TB_PARTTIONS; + String sql = "SELECT A." + COL_PARTITIONS_PK + ", A." + COL_TABLES_PK + ", A.PARTITION_NAME, " + + " A.PATH FROM " + TB_PARTTIONS + " A JOIN " + TB_TABLES + " B " + + " ON A." + COL_TABLES_PK + " = B." + COL_TABLES_PK + + " WHERE A." + COL_PARTITIONS_PK + " > 0 "; conn = getConnection(); stmt = conn.createStatement(); @@ -2268,7 +2270,11 @@ public List getAllPartitionKeys() throws CatalogExcepti List partitions = new ArrayList(); try { - String sql = "SELECT " + COL_PARTITIONS_PK + ", COLUMN_NAME, PARTITION_VALUE FROM " + TB_PARTTION_KEYS; + String sql = " SELECT C." + COL_PARTITIONS_PK + ", C.COLUMN_NAME, C.PARTITION_VALUE " + + " FROM " + TB_TABLES + " A " + + " JOIN " + TB_PARTTIONS + " B ON A." + COL_TABLES_PK + " = B." + COL_TABLES_PK + + " AND B." + COL_PARTITIONS_PK + " > 0 " + + " JOIN " + TB_PARTTION_KEYS + " C ON B." + COL_PARTITIONS_PK + " = C." + COL_PARTITIONS_PK; conn = getConnection(); stmt = conn.createStatement(); From 95e1f4ff065f6287c406412a6891cfafe7cc79b4 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 4 Aug 2015 17:27:00 +0900 Subject: [PATCH 07/25] Remove debug logs --- .../apache/tajo/engine/query/TestAlterTable.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 9e7da2fce0..49941fc239 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -128,17 +128,8 @@ public final void testAlterTableAddPartition() throws Exception { List allPartitions = catalog.getAllPartitions(); List resultPartitions = TUtil.newList(); - System.out.println("### tablePath:" + retrieved.getUri().toString()); - System.out.println("### tableId:" + tableId); - int partitionId = 0; for (CatalogProtos.TablePartitionProto partition : allPartitions) { - System.out.println("### partition - partitionId:" + partition.getPartitionId() - + ", tid:" + partition.getTid() - + ", name:" + partition.getPartitionName() - + ", path:" + partition.getPath() - ); - if (partition.getTid() == tableId && partition.getPartitionName().equals("col3=1/col4=2") && partition.getPath().equals(retrieved.getUri().toString() + "/col3=1/col4=2") @@ -154,10 +145,6 @@ public final void testAlterTableAddPartition() throws Exception { List resultPartitionKeys = TUtil.newList(); for (CatalogProtos.TablePartitionKeysProto partitionKey: tablePartitionKeys) { - System.out.println("### partitionKeys - partitionId:" + partitionKey.getPartitionId() - + ", column:" + partitionKey.getColumnName() - + ", value:" + partitionKey.getPartitionValue() - ); if (partitionKey.getPartitionId() == partitionId && (partitionKey.getColumnName().equals("col3") && partitionKey.getPartitionValue().equals("1") || partitionKey.getColumnName().equals("col4") && partitionKey.getPartitionValue().equals("2"))) { From d6c9390a8cda54c86d81f27dcdfb51935d662b3c Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 17 Aug 2015 12:55:48 +0900 Subject: [PATCH 08/25] Remove unnecessary join clause and exception. --- .../org/apache/tajo/catalog/CatalogServer.java | 4 ++-- .../tajo/catalog/store/AbstractDBStore.java | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 4dc14e3131..00ca485f7f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -1044,8 +1044,8 @@ public ReturnState addPartitions(RpcController controller, AddPartitionsProto re } @Override - public GetTablePartitionKeysResponse getAllPartitionKeys(RpcController controller, - NullProto request) throws ServiceException { + public GetTablePartitionKeysResponse getAllPartitionKeys(RpcController controller, NullProto request) throws + ServiceException { rlock.lock(); try { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index ef26ce8ca9..4bc4d50541 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2138,10 +2138,10 @@ public List getAllPartitions() { List partitions = new ArrayList(); try { - String sql = "SELECT A." + COL_PARTITIONS_PK + ", A." + COL_TABLES_PK + ", A.PARTITION_NAME, " + - " A.PATH FROM " + TB_PARTTIONS + " A JOIN " + TB_TABLES + " B " - + " ON A." + COL_TABLES_PK + " = B." + COL_TABLES_PK - + " WHERE A." + COL_PARTITIONS_PK + " > 0 "; + String sql = "SELECT A." + COL_PARTITIONS_PK + ", A." + COL_TABLES_PK + ", A.PARTITION_NAME, A.PATH " + + " FROM " + TB_PARTTIONS + " A " + + " WHERE A." + COL_PARTITIONS_PK + " > 0 " + + " AND A." + COL_TABLES_PK + " > 0 "; conn = getConnection(); stmt = conn.createStatement(); @@ -2288,11 +2288,11 @@ public List getAllPartitionKeys() { List partitions = new ArrayList(); try { - String sql = " SELECT C." + COL_PARTITIONS_PK + ", C.COLUMN_NAME, C.PARTITION_VALUE " + - " FROM " + TB_TABLES + " A " + - " JOIN " + TB_PARTTIONS + " B ON A." + COL_TABLES_PK + " = B." + COL_TABLES_PK - + " AND B." + COL_PARTITIONS_PK + " > 0 " + - " JOIN " + TB_PARTTION_KEYS + " C ON B." + COL_PARTITIONS_PK + " = C." + COL_PARTITIONS_PK; + String sql = " SELECT A." + COL_PARTITIONS_PK + ", A.COLUMN_NAME, A.PARTITION_VALUE " + + " FROM " + TB_PARTTION_KEYS + " A " + + " WHERE A." + COL_PARTITIONS_PK + " > 0 "+ + " AND A.COLUMN_NAME IS NOT NULL " + + " AND A.PARTITION_VALUE IS NOT NULL "; conn = getConnection(); stmt = conn.createStatement(); From 6f5abed5c1ccdd393c271c0b0fcd0001f71c0a11 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 17 Aug 2015 13:05:18 +0900 Subject: [PATCH 09/25] Rename TablePartitionKeysProto to TablePartitionKeyProto --- .../java/org/apache/tajo/catalog/AbstractCatalogClient.java | 2 +- .../src/main/proto/CatalogProtocol.proto | 2 +- .../main/java/org/apache/tajo/catalog/CatalogService.java | 2 +- .../tajo-catalog-common/src/main/proto/CatalogProtos.proto | 2 +- .../org/apache/tajo/catalog/store/HiveCatalogStore.java | 2 +- .../java/org/apache/tajo/catalog/store/AbstractDBStore.java | 6 +++--- .../java/org/apache/tajo/catalog/store/CatalogStore.java | 2 +- .../main/java/org/apache/tajo/catalog/store/MemStore.java | 6 +++--- .../java/org/apache/tajo/engine/query/TestAlterTable.java | 6 +++--- .../master/exec/NonForwardQueryResultSystemScanner.java | 4 ++-- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 2dc8804283..7ddcce7d2b 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -491,7 +491,7 @@ public void addPartitions(String databaseName, String tableName, List getAllPartitionKeys() { + public List getAllPartitionKeys() { try { final BlockingInterface stub = getStub(); final GetTablePartitionKeysResponse response = stub.getAllPartitionKeys(null, ProtoUtil.NULL_PROTO); diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index e462794b6c..dde8e081f0 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -91,7 +91,7 @@ message GetTablePartitionsResponse { message GetTablePartitionKeysResponse { required ReturnState state = 1; - repeated TablePartitionKeysProto partKey = 2; + repeated TablePartitionKeyProto partKey = 2; } service CatalogProtocolService { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index c0a21c8bb6..84c0bf406e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -169,7 +169,7 @@ PartitionDescProto getPartition(String databaseName, String tableName, String pa void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws UndefinedTableException, DuplicatePartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException; - List getAllPartitionKeys(); + List getAllPartitionKeys(); boolean createIndex(IndexDesc index); diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 00a46821e9..dfe3153ec9 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -152,7 +152,7 @@ message TablePartitionProto { optional string path = 4; } -message TablePartitionKeysProto { +message TablePartitionKeyProto { required int32 partition_id = 1; required string columnName = 2; required string partitionValue = 3; diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 3043ab6a0e..bae6b37633 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -1006,7 +1006,7 @@ public List getAllPartitions() { } @Override - public List getAllPartitionKeys() { + public List getAllPartitionKeys() { throw new UnsupportedOperationException(); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 4bc4d50541..8cf71c7c6f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2280,12 +2280,12 @@ public void addPartitions(String databaseName, String tableName, List getAllPartitionKeys() { + public List getAllPartitionKeys() { Connection conn = null; Statement stmt = null; ResultSet resultSet = null; - List partitions = new ArrayList(); + List partitions = new ArrayList(); try { String sql = " SELECT A." + COL_PARTITIONS_PK + ", A.COLUMN_NAME, A.PARTITION_VALUE " + @@ -2298,7 +2298,7 @@ public List getAllPartitionKeys() { stmt = conn.createStatement(); resultSet = stmt.executeQuery(sql); while (resultSet.next()) { - TablePartitionKeysProto.Builder builder = TablePartitionKeysProto.newBuilder(); + TablePartitionKeyProto.Builder builder = TablePartitionKeyProto.newBuilder(); builder.setPartitionId(resultSet.getInt(COL_PARTITIONS_PK)); builder.setColumnName(resultSet.getString("COLUMN_NAME")); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 5451c667ee..a458b757e3 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -107,7 +107,7 @@ CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableN List getAllPartitions(); - List getAllPartitionKeys(); + List getAllPartitionKeys(); void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws UndefinedDatabaseException, diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 8818421f92..86f0da7e18 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -604,8 +604,8 @@ public List getAllPartitions() { return protos; } - public List getAllPartitionKeys() { - List protos = new ArrayList(); + public List getAllPartitionKeys() { + List protos = new ArrayList(); int partitionId = 0; Set partitionTables = partitions.keySet(); @@ -615,7 +615,7 @@ public List getAllPartitionKeys() { CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); for (PartitionKeyProto partitionKey : partitionDescProto.getPartitionKeysList()) { - TablePartitionKeysProto.Builder builder = TablePartitionKeysProto.newBuilder(); + TablePartitionKeyProto.Builder builder = TablePartitionKeyProto.newBuilder(); builder.setColumnName(partitionKey.getColumnName()); builder.setPartitionValue(partitionKey.getPartitionValue()); builder.setPartitionId(partitionId); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 49941fc239..f450704da6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -141,10 +141,10 @@ public final void testAlterTableAddPartition() throws Exception { assertEquals(resultPartitions.size(), 1); assertEquals(resultPartitions.get(0).getPartitionName(), "col3=1/col4=2"); - List tablePartitionKeys = catalog.getAllPartitionKeys(); - List resultPartitionKeys = TUtil.newList(); + List tablePartitionKeys = catalog.getAllPartitionKeys(); + List resultPartitionKeys = TUtil.newList(); - for (CatalogProtos.TablePartitionKeysProto partitionKey: tablePartitionKeys) { + for (CatalogProtos.TablePartitionKeyProto partitionKey: tablePartitionKeys) { if (partitionKey.getPartitionId() == partitionId && (partitionKey.getColumnName().equals("col3") && partitionKey.getPartitionValue().equals("1") || partitionKey.getColumnName().equals("col4") && partitionKey.getPartitionValue().equals("2"))) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 3855852e5a..c41a671493 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -420,12 +420,12 @@ private List getAllPartitions(Schema outSchema) { } private List getAllPartitionKeys(Schema outSchema) { - List partitionKeyList = masterContext.getCatalog().getAllPartitionKeys(); + List partitionKeyList = masterContext.getCatalog().getAllPartitionKeys(); List tuples = new ArrayList(partitionKeyList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; - for (TablePartitionKeysProto partitionKey: partitionKeyList) { + for (TablePartitionKeyProto partitionKey: partitionKeyList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { From 4f81a28c9ab58500386702478878a3f0aeccffce Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 18 Aug 2015 08:11:29 +0900 Subject: [PATCH 10/25] Remove partitionId in MemStore --- .../apache/tajo/catalog/store/MemStore.java | 30 +++------ .../tajo/engine/query/TestAlterTable.java | 66 +++---------------- 2 files changed, 20 insertions(+), 76 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 86f0da7e18..43dd4219d2 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -574,19 +574,10 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String } public List getAllPartitions() { - int tableId = 0, partitionId = 0; - List tables = getAllTables(); List protos = new ArrayList(); - - Set partitionTables = partitions.keySet(); - for (String partitionTable : partitionTables) { - for (TableDescriptorProto table : tables) { - if (table.getName().equals(partitionTable)) { - tableId = table.getTid(); - } - } - - Map entryMap = partitions.get(partitionTable); + Set tables = partitions.keySet(); + for (String table : tables) { + Map entryMap = partitions.get(table); for (Map.Entry proto : entryMap.entrySet()) { CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); @@ -594,11 +585,14 @@ public List getAllPartitions() { builder.setPartitionName(partitionDescProto.getPartitionName()); builder.setPath(partitionDescProto.getPath()); - builder.setPartitionId(partitionId); - builder.setTid(tableId); + + // PARTITION_ID and TID is always necessary variables. In other CatalogStore excepting MemStore, + // all partitions would have PARTITION_ID and TID. But MemStore doesn't contain these variable values because + // it is implemented for test purpose. Thus, we need to set each variables to 0. + builder.setPartitionId(0); + builder.setTid(0); protos.add(builder.build()); - partitionId++; } } return protos; @@ -606,8 +600,6 @@ public List getAllPartitions() { public List getAllPartitionKeys() { List protos = new ArrayList(); - int partitionId = 0; - Set partitionTables = partitions.keySet(); for (String partitionTable : partitionTables) { Map entryMap = partitions.get(partitionTable); @@ -618,11 +610,9 @@ public List getAllPartitionKeys() { TablePartitionKeyProto.Builder builder = TablePartitionKeyProto.newBuilder(); builder.setColumnName(partitionKey.getColumnName()); builder.setPartitionValue(partitionKey.getPartitionValue()); - builder.setPartitionId(partitionId); + builder.setPartitionId(0); protos.add(builder.build()); } - - partitionId++; } } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index f450704da6..51e96ed98c 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -107,74 +107,28 @@ public final void testAlterTableAddPartition() throws Exception { assertTrue(fs.exists(partitionPath)); assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); - List allDatabases = catalog.getAllDatabases(); - int dbId = -1; - for (CatalogProtos.DatabaseProto database : allDatabases) { - if (database.getName().equals(getCurrentDatabase())) { - dbId = database.getId(); - } - } - assertNotEquals(dbId, -1); - - int tableId = -1; - List allTables = catalog.getAllTables(); - for(CatalogProtos.TableDescriptorProto table : allTables) { - if (table.getDbId() == dbId && table.getName().equals(simpleTableName)) { - tableId = table.getTid(); - } - } - assertNotEquals(tableId, -1); - + boolean existPartition = false; List allPartitions = catalog.getAllPartitions(); - List resultPartitions = TUtil.newList(); - - int partitionId = 0; for (CatalogProtos.TablePartitionProto partition : allPartitions) { - if (partition.getTid() == tableId - && partition.getPartitionName().equals("col3=1/col4=2") + if (partition.getPartitionName().equals("col3=1/col4=2") && partition.getPath().equals(retrieved.getUri().toString() + "/col3=1/col4=2") ){ - resultPartitions.add(partition); - partitionId = partition.getPartitionId(); + existPartition = true; + break; } } - assertEquals(resultPartitions.size(), 1); - assertEquals(resultPartitions.get(0).getPartitionName(), "col3=1/col4=2"); + assertTrue(existPartition); + boolean existPartitionKey = false; List tablePartitionKeys = catalog.getAllPartitionKeys(); - List resultPartitionKeys = TUtil.newList(); - for (CatalogProtos.TablePartitionKeyProto partitionKey: tablePartitionKeys) { - if (partitionKey.getPartitionId() == partitionId - && (partitionKey.getColumnName().equals("col3") && partitionKey.getPartitionValue().equals("1") + if ((partitionKey.getColumnName().equals("col3") && partitionKey.getPartitionValue().equals("1") || partitionKey.getColumnName().equals("col4") && partitionKey.getPartitionValue().equals("2"))) { - resultPartitionKeys.add(partitionKey); + existPartitionKey = true; + break; } } - assertEquals(resultPartitionKeys.size(), 2); - assertEquals(resultPartitionKeys.get(0).getColumnName(), "col3"); - assertEquals(resultPartitionKeys.get(0).getPartitionValue(), "1"); - assertEquals(resultPartitionKeys.get(1).getColumnName(), "col4"); - assertEquals(resultPartitionKeys.get(1).getPartitionValue(), "2"); - - ResultSet resultSet = executeString("SELECT partition_name FROM INFORMATION_SCHEMA.PARTITIONS " - + " WHERE partition_id = " + partitionId); - - String actualResult = resultSetToString(resultSet); - String expectedResult = "partition_name\n" + - "-------------------------------\n" + - "col3=1/col4=2\n"; - assertEquals(expectedResult, actualResult); - - resultSet = executeString("SELECT column_name,partition_value FROM INFORMATION_SCHEMA.PARTITION_KEYS" + - " WHERE partition_id = " + partitionId); - - actualResult = resultSetToString(resultSet); - expectedResult = "column_name,partition_value\n" + - "-------------------------------\n" + - "col3,1\n" + - "col4,2\n"; - assertEquals(expectedResult, actualResult); + assertTrue(existPartitionKey); executeDDL("alter_table_drop_partition1.sql", null); executeDDL("alter_table_drop_partition2.sql", null); From 0bb89b6b25e4eac6cea691747d1388d746f9f5da Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 20 Aug 2015 11:50:22 +0900 Subject: [PATCH 11/25] Update PB for TablePartition --- .../java/org/apache/tajo/catalog/AbstractCatalogClient.java | 4 ++-- .../tajo-catalog-client/src/main/proto/CatalogProtocol.proto | 4 ++-- .../tajo-catalog-common/src/main/proto/CatalogProtos.proto | 4 ++-- .../src/main/java/org/apache/tajo/catalog/CatalogServer.java | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 7ddcce7d2b..c4ba9f4f63 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -452,7 +452,7 @@ public List getAllPartitions() { final GetTablePartitionsResponse response = stub.getAllPartitions(null, ProtoUtil.NULL_PROTO); ensureOk(response.getState()); - return response.getPartList(); + return response.getPartitionList(); } catch (ServiceException e) { throw new RuntimeException(e); @@ -497,7 +497,7 @@ public List getAllPartitionKeys() { final GetTablePartitionKeysResponse response = stub.getAllPartitionKeys(null, ProtoUtil.NULL_PROTO); ensureOk(response.getState()); - return response.getPartKeyList(); + return response.getPartitionKeyList(); } catch (ServiceException e) { throw new RuntimeException(e); diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index dde8e081f0..321531c789 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -86,12 +86,12 @@ message GetPartitionsResponse { message GetTablePartitionsResponse { required ReturnState state = 1; - repeated TablePartitionProto part = 2; + repeated TablePartitionProto partition = 2; } message GetTablePartitionKeysResponse { required ReturnState state = 1; - repeated TablePartitionKeyProto partKey = 2; + repeated TablePartitionKeyProto partitionKey = 2; } service CatalogProtocolService { diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index dfe3153ec9..f653f6e1f6 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -155,7 +155,7 @@ message TablePartitionProto { message TablePartitionKeyProto { required int32 partition_id = 1; required string columnName = 2; - required string partitionValue = 3; + optional string partitionValue = 3; } message GetFunctionsResponse { @@ -259,7 +259,7 @@ message PartitionDescProto { message PartitionKeyProto { required string columnName = 1; optional string parentColumnName = 2; - required string partitionValue = 3; + optional string partitionValue = 3; } message PartitionIdentifierProto { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 00ca485f7f..13ac7fb89b 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -1007,7 +1007,7 @@ public GetTablePartitionsResponse getAllPartitions(RpcController controller, Nul try { return GetTablePartitionsResponse.newBuilder() .setState(OK) - .addAllPart(store.getAllPartitions()) + .addAllPartition(store.getAllPartitions()) .build(); } catch (Throwable t) { @@ -1051,7 +1051,7 @@ public GetTablePartitionKeysResponse getAllPartitionKeys(RpcController controlle try { return GetTablePartitionKeysResponse.newBuilder() .setState(OK) - .addAllPartKey(store.getAllPartitionKeys()) + .addAllPartitionKey(store.getAllPartitionKeys()) .build(); } catch (Throwable t) { From 2873d05e936780b252911f3e59e3a57f7375f471 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 20 Aug 2015 11:54:01 +0900 Subject: [PATCH 12/25] Remove ServiceException --- .../main/java/org/apache/tajo/catalog/CatalogServer.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 13ac7fb89b..9935a522a2 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -999,9 +999,7 @@ public GetPartitionsResponse getPartitionsByTableName(RpcController controller, } @Override - public GetTablePartitionsResponse getAllPartitions(RpcController controller, NullProto request) - throws ServiceException { - + public GetTablePartitionsResponse getAllPartitions(RpcController controller, NullProto request) { rlock.lock(); try { @@ -1044,8 +1042,7 @@ public ReturnState addPartitions(RpcController controller, AddPartitionsProto re } @Override - public GetTablePartitionKeysResponse getAllPartitionKeys(RpcController controller, NullProto request) throws - ServiceException { + public GetTablePartitionKeysResponse getAllPartitionKeys(RpcController controller, NullProto request) { rlock.lock(); try { From b77a1fe360aee60adbce85fd061209b74f639923 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 20 Aug 2015 11:54:53 +0900 Subject: [PATCH 13/25] Remove unnecessary filters --- .../org/apache/tajo/catalog/store/AbstractDBStore.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 8cf71c7c6f..60027d34cc 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2139,9 +2139,7 @@ public List getAllPartitions() { try { String sql = "SELECT A." + COL_PARTITIONS_PK + ", A." + COL_TABLES_PK + ", A.PARTITION_NAME, A.PATH " - + " FROM " + TB_PARTTIONS + " A " - + " WHERE A." + COL_PARTITIONS_PK + " > 0 " - + " AND A." + COL_TABLES_PK + " > 0 "; + + " FROM " + TB_PARTTIONS + " A "; conn = getConnection(); stmt = conn.createStatement(); @@ -2289,10 +2287,7 @@ public List getAllPartitionKeys() { try { String sql = " SELECT A." + COL_PARTITIONS_PK + ", A.COLUMN_NAME, A.PARTITION_VALUE " + - " FROM " + TB_PARTTION_KEYS + " A " + - " WHERE A." + COL_PARTITIONS_PK + " > 0 "+ - " AND A.COLUMN_NAME IS NOT NULL " + - " AND A.PARTITION_VALUE IS NOT NULL "; + " FROM " + TB_PARTTION_KEYS + " A "; conn = getConnection(); stmt = conn.createStatement(); From 868b72129f685249fe9b0e44a3fdcddd174cde78 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 20 Aug 2015 12:16:48 +0900 Subject: [PATCH 14/25] Remove TablePartitionKeyProto --- .../org/apache/tajo/catalog/AbstractCatalogClient.java | 4 ++-- .../src/main/proto/CatalogProtocol.proto | 6 +++--- .../main/java/org/apache/tajo/catalog/CatalogService.java | 2 +- .../tajo-catalog-common/src/main/proto/CatalogProtos.proto | 7 +------ .../org/apache/tajo/catalog/store/HiveCatalogStore.java | 2 +- .../main/java/org/apache/tajo/catalog/CatalogServer.java | 6 +++--- .../org/apache/tajo/catalog/store/AbstractDBStore.java | 6 +++--- .../java/org/apache/tajo/catalog/store/CatalogStore.java | 2 +- .../main/java/org/apache/tajo/catalog/store/MemStore.java | 6 +++--- .../java/org/apache/tajo/engine/query/TestAlterTable.java | 4 ++-- .../master/exec/NonForwardQueryResultSystemScanner.java | 4 ++-- 11 files changed, 22 insertions(+), 27 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index c4ba9f4f63..8abc2f575c 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -491,10 +491,10 @@ public void addPartitions(String databaseName, String tableName, List getAllPartitionKeys() { + public List getAllPartitionKeys() { try { final BlockingInterface stub = getStub(); - final GetTablePartitionKeysResponse response = stub.getAllPartitionKeys(null, ProtoUtil.NULL_PROTO); + final GetPartitionKeysResponse response = stub.getAllPartitionKeys(null, ProtoUtil.NULL_PROTO); ensureOk(response.getState()); return response.getPartitionKeyList(); diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index 321531c789..a5c4d27bec 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -89,9 +89,9 @@ message GetTablePartitionsResponse { repeated TablePartitionProto partition = 2; } -message GetTablePartitionKeysResponse { +message GetPartitionKeysResponse { required ReturnState state = 1; - repeated TablePartitionKeyProto partitionKey = 2; + repeated PartitionKeyProto partitionKey = 2; } service CatalogProtocolService { @@ -129,7 +129,7 @@ service CatalogProtocolService { rpc getPartitionsByTableName(PartitionIdentifierProto) returns (GetPartitionsResponse); rpc getAllPartitions(NullProto) returns (GetTablePartitionsResponse); rpc addPartitions(AddPartitionsProto) returns (ReturnState); - rpc getAllPartitionKeys(NullProto) returns (GetTablePartitionKeysResponse); + rpc getAllPartitionKeys(NullProto) returns (GetPartitionKeysResponse); rpc createIndex(IndexDescProto) returns (ReturnState); rpc dropIndex(IndexNameProto) returns (ReturnState); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index 84c0bf406e..af9cc7f2ce 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -169,7 +169,7 @@ PartitionDescProto getPartition(String databaseName, String tableName, String pa void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws UndefinedTableException, DuplicatePartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException; - List getAllPartitionKeys(); + List getAllPartitionKeys(); boolean createIndex(IndexDesc index); diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index f653f6e1f6..927b1857b0 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -152,12 +152,6 @@ message TablePartitionProto { optional string path = 4; } -message TablePartitionKeyProto { - required int32 partition_id = 1; - required string columnName = 2; - optional string partitionValue = 3; -} - message GetFunctionsResponse { required ReturnState state = 1; repeated FunctionDescProto functionDesc = 2; @@ -260,6 +254,7 @@ message PartitionKeyProto { required string columnName = 1; optional string parentColumnName = 2; optional string partitionValue = 3; + optional int32 partition_id = 4; } message PartitionIdentifierProto { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 61ca44b153..d23a0a5d4f 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -1010,7 +1010,7 @@ public List getAllPartitions() { } @Override - public List getAllPartitionKeys() { + public List getAllPartitionKeys() { throw new UnsupportedOperationException(); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 9935a522a2..67bf25bafc 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -1042,11 +1042,11 @@ public ReturnState addPartitions(RpcController controller, AddPartitionsProto re } @Override - public GetTablePartitionKeysResponse getAllPartitionKeys(RpcController controller, NullProto request) { + public GetPartitionKeysResponse getAllPartitionKeys(RpcController controller, NullProto request) { rlock.lock(); try { - return GetTablePartitionKeysResponse.newBuilder() + return GetPartitionKeysResponse.newBuilder() .setState(OK) .addAllPartitionKey(store.getAllPartitionKeys()) .build(); @@ -1054,7 +1054,7 @@ public GetTablePartitionKeysResponse getAllPartitionKeys(RpcController controlle } catch (Throwable t) { printStackTraceIfError(LOG, t); - return GetTablePartitionKeysResponse.newBuilder() + return GetPartitionKeysResponse.newBuilder() .setState(returnError(t)) .build(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 60027d34cc..37f6c44781 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2278,12 +2278,12 @@ public void addPartitions(String databaseName, String tableName, List getAllPartitionKeys() { + public List getAllPartitionKeys() { Connection conn = null; Statement stmt = null; ResultSet resultSet = null; - List partitions = new ArrayList(); + List partitions = new ArrayList(); try { String sql = " SELECT A." + COL_PARTITIONS_PK + ", A.COLUMN_NAME, A.PARTITION_VALUE " + @@ -2293,7 +2293,7 @@ public List getAllPartitionKeys() { stmt = conn.createStatement(); resultSet = stmt.executeQuery(sql); while (resultSet.next()) { - TablePartitionKeyProto.Builder builder = TablePartitionKeyProto.newBuilder(); + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setPartitionId(resultSet.getInt(COL_PARTITIONS_PK)); builder.setColumnName(resultSet.getString("COLUMN_NAME")); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index a458b757e3..b707f477b9 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -107,7 +107,7 @@ CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableN List getAllPartitions(); - List getAllPartitionKeys(); + List getAllPartitionKeys(); void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws UndefinedDatabaseException, diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 43dd4219d2..541a70c197 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -598,8 +598,8 @@ public List getAllPartitions() { return protos; } - public List getAllPartitionKeys() { - List protos = new ArrayList(); + public List getAllPartitionKeys() { + List protos = new ArrayList(); Set partitionTables = partitions.keySet(); for (String partitionTable : partitionTables) { Map entryMap = partitions.get(partitionTable); @@ -607,7 +607,7 @@ public List getAllPartitionKeys() { CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); for (PartitionKeyProto partitionKey : partitionDescProto.getPartitionKeysList()) { - TablePartitionKeyProto.Builder builder = TablePartitionKeyProto.newBuilder(); + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(partitionKey.getColumnName()); builder.setPartitionValue(partitionKey.getPartitionValue()); builder.setPartitionId(0); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 51e96ed98c..f3d4862bfa 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -120,8 +120,8 @@ public final void testAlterTableAddPartition() throws Exception { assertTrue(existPartition); boolean existPartitionKey = false; - List tablePartitionKeys = catalog.getAllPartitionKeys(); - for (CatalogProtos.TablePartitionKeyProto partitionKey: tablePartitionKeys) { + List tablePartitionKeys = catalog.getAllPartitionKeys(); + for (CatalogProtos.PartitionKeyProto partitionKey: tablePartitionKeys) { if ((partitionKey.getColumnName().equals("col3") && partitionKey.getPartitionValue().equals("1") || partitionKey.getColumnName().equals("col4") && partitionKey.getPartitionValue().equals("2"))) { existPartitionKey = true; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index c41a671493..944c40a300 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -420,12 +420,12 @@ private List getAllPartitions(Schema outSchema) { } private List getAllPartitionKeys(Schema outSchema) { - List partitionKeyList = masterContext.getCatalog().getAllPartitionKeys(); + List partitionKeyList = masterContext.getCatalog().getAllPartitionKeys(); List tuples = new ArrayList(partitionKeyList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; - for (TablePartitionKeyProto partitionKey: partitionKeyList) { + for (PartitionKeyProto partitionKey: partitionKeyList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { From 124992edd12c3c70f1fea683d1a347c9e279a8b7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 20 Aug 2015 12:31:24 +0900 Subject: [PATCH 15/25] Remove TablePartitionProto --- .../apache/tajo/catalog/AbstractCatalogClient.java | 4 ++-- .../src/main/proto/CatalogProtocol.proto | 7 +------ .../org/apache/tajo/catalog/CatalogService.java | 2 +- .../src/main/proto/CatalogProtos.proto | 10 ++-------- .../tajo/catalog/store/HiveCatalogStore.java | 2 +- .../org/apache/tajo/catalog/CatalogServer.java | 6 +++--- .../apache/tajo/catalog/store/AbstractDBStore.java | 14 +++++++------- .../apache/tajo/catalog/store/CatalogStore.java | 2 +- .../org/apache/tajo/catalog/store/MemStore.java | 6 +++--- .../apache/tajo/engine/query/TestAlterTable.java | 4 ++-- .../exec/NonForwardQueryResultSystemScanner.java | 4 ++-- 11 files changed, 25 insertions(+), 36 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 8abc2f575c..3901bc878f 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -446,10 +446,10 @@ public final List getPartitions(final String databaseName, f } @Override - public List getAllPartitions() { + public List getAllPartitions() { try { final BlockingInterface stub = getStub(); - final GetTablePartitionsResponse response = stub.getAllPartitions(null, ProtoUtil.NULL_PROTO); + final GetPartitionsResponse response = stub.getAllPartitions(null, ProtoUtil.NULL_PROTO); ensureOk(response.getState()); return response.getPartitionList(); diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index a5c4d27bec..e57fddbc67 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -84,11 +84,6 @@ message GetPartitionsResponse { repeated PartitionDescProto partition = 2; } -message GetTablePartitionsResponse { - required ReturnState state = 1; - repeated TablePartitionProto partition = 2; -} - message GetPartitionKeysResponse { required ReturnState state = 1; repeated PartitionKeyProto partitionKey = 2; @@ -127,7 +122,7 @@ service CatalogProtocolService { rpc getPartitionByPartitionName(PartitionIdentifierProto) returns (GetPartitionDescResponse); rpc getPartitionsByTableName(PartitionIdentifierProto) returns (GetPartitionsResponse); - rpc getAllPartitions(NullProto) returns (GetTablePartitionsResponse); + rpc getAllPartitions(NullProto) returns (GetPartitionsResponse); rpc addPartitions(AddPartitionsProto) returns (ReturnState); rpc getAllPartitionKeys(NullProto) returns (GetPartitionKeysResponse); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index af9cc7f2ce..8546bb8eb0 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -164,7 +164,7 @@ PartitionDescProto getPartition(String databaseName, String tableName, String pa List getPartitions(String databaseName, String tableName); - List getAllPartitions(); + List getAllPartitions(); void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws UndefinedTableException, DuplicatePartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException; diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 927b1857b0..2233f5d154 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -145,13 +145,6 @@ message TableOptionProto { required KeyValueProto keyval = 2; } -message TablePartitionProto { - required int32 partition_id = 1; - required int32 tid = 2; - optional string partitionName = 3; - optional string path = 4; -} - message GetFunctionsResponse { required ReturnState state = 1; repeated FunctionDescProto functionDesc = 2; @@ -247,7 +240,8 @@ message PartitionDescProto { required string partitionName = 1; repeated PartitionKeyProto partitionKeys = 2; optional string path = 3; - optional int32 id = 4; + optional int32 partition_id = 4; + optional int32 tid = 5; } message PartitionKeyProto { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index d23a0a5d4f..d9a7b172b7 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -1005,7 +1005,7 @@ public List getAllIndexes() { } @Override - public List getAllPartitions() { + public List getAllPartitions() { throw new UnsupportedOperationException(); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 67bf25bafc..f093c8044c 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -999,11 +999,11 @@ public GetPartitionsResponse getPartitionsByTableName(RpcController controller, } @Override - public GetTablePartitionsResponse getAllPartitions(RpcController controller, NullProto request) { + public GetPartitionsResponse getAllPartitions(RpcController controller, NullProto request) { rlock.lock(); try { - return GetTablePartitionsResponse.newBuilder() + return GetPartitionsResponse.newBuilder() .setState(OK) .addAllPartition(store.getAllPartitions()) .build(); @@ -1011,7 +1011,7 @@ public GetTablePartitionsResponse getAllPartitions(RpcController controller, Nul } catch (Throwable t) { printStackTraceIfError(LOG, t); - return GetTablePartitionsResponse.newBuilder() + return GetPartitionsResponse.newBuilder() .setState(returnError(t)) .build(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 37f6c44781..415c91addb 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1041,7 +1041,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) if (partitionDesc == null) { throw new UndefinedPartitionException(partitionName); } - dropPartition(partitionDesc.getId()); + dropPartition(partitionDesc.getPartitionId()); break; case SET_PROPERTY: setProperties(tableId, alterTableDescProto.getParams()); @@ -2045,7 +2045,7 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String if (res.next()) { builder = PartitionDescProto.newBuilder(); - builder.setId(res.getInt(COL_PARTITIONS_PK)); + builder.setPartitionId(res.getInt(COL_PARTITIONS_PK)); builder.setPath(res.getString("PATH")); builder.setPartitionName(partitionName); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); @@ -2130,12 +2130,12 @@ public List getPartitions(String databaseName, String tableN } @Override - public List getAllPartitions() { + public List getAllPartitions() { Connection conn = null; Statement stmt = null; ResultSet resultSet = null; - List partitions = new ArrayList(); + List partitions = new ArrayList(); try { String sql = "SELECT A." + COL_PARTITIONS_PK + ", A." + COL_TABLES_PK + ", A.PARTITION_NAME, A.PATH " @@ -2145,7 +2145,7 @@ public List getAllPartitions() { stmt = conn.createStatement(); resultSet = stmt.executeQuery(sql); while (resultSet.next()) { - TablePartitionProto.Builder builder = TablePartitionProto.newBuilder(); + PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionId(resultSet.getInt(COL_PARTITIONS_PK)); builder.setTid(resultSet.getInt(COL_TABLES_PK)); @@ -2205,11 +2205,11 @@ public void addPartitions(String databaseName, String tableName, List getAllPartitions(); + List getAllPartitions(); List getAllPartitionKeys(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 541a70c197..591cfb5e2c 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -573,15 +573,15 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String } } - public List getAllPartitions() { - List protos = new ArrayList(); + public List getAllPartitions() { + List protos = new ArrayList(); Set tables = partitions.keySet(); for (String table : tables) { Map entryMap = partitions.get(table); for (Map.Entry proto : entryMap.entrySet()) { CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); - TablePartitionProto.Builder builder = TablePartitionProto.newBuilder(); + PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partitionDescProto.getPartitionName()); builder.setPath(partitionDescProto.getPath()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index f3d4862bfa..0919a3f8de 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -108,8 +108,8 @@ public final void testAlterTableAddPartition() throws Exception { assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); boolean existPartition = false; - List allPartitions = catalog.getAllPartitions(); - for (CatalogProtos.TablePartitionProto partition : allPartitions) { + List allPartitions = catalog.getAllPartitions(); + for (CatalogProtos.PartitionDescProto partition : allPartitions) { if (partition.getPartitionName().equals("col3=1/col4=2") && partition.getPath().equals(retrieved.getUri().toString() + "/col3=1/col4=2") ){ diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 944c40a300..c1d4e258da 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -387,12 +387,12 @@ private List getAllTableStats(Schema outSchema) { } private List getAllPartitions(Schema outSchema) { - List partitionList = masterContext.getCatalog().getAllPartitions(); + List partitionList = masterContext.getCatalog().getAllPartitions(); List tuples = new ArrayList(partitionList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; - for (TablePartitionProto partition: partitionList) { + for (PartitionDescProto partition: partitionList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { From c4ffc05fd9cb64069b9bdb11da6d94a097356933 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 20 Aug 2015 13:50:02 +0900 Subject: [PATCH 16/25] Move test cases to TestNonForwardQueryResultSystemScanner --- .../apache/tajo/catalog/store/MemStore.java | 31 +++--- .../tajo/engine/query/TestAlterTable.java | 31 +----- ...estNonForwardQueryResultSystemScanner.java | 98 +++++++++++++++++++ .../alter_table_add_partition1.sql | 1 + .../alter_table_add_partition2.sql | 1 + .../create_partitioned_table.sql | 1 + .../drop_partitioned_table.sql | 1 + 7 files changed, 125 insertions(+), 39 deletions(-) create mode 100644 tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition1.sql create mode 100644 tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition2.sql create mode 100644 tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/create_partitioned_table.sql create mode 100644 tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/drop_partitioned_table.sql diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 591cfb5e2c..ff9377aaf1 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -574,10 +574,19 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String } public List getAllPartitions() { + int tableId = 0, partitionId = 0; + List tables = getAllTables(); List protos = new ArrayList(); - Set tables = partitions.keySet(); - for (String table : tables) { - Map entryMap = partitions.get(table); + + Set partitionTables = partitions.keySet(); + for (String partitionTable : partitionTables) { + for (TableDescriptorProto table : tables) { + if (table.getName().equals(partitionTable)) { + tableId = table.getTid(); + } + } + + Map entryMap = partitions.get(partitionTable); for (Map.Entry proto : entryMap.entrySet()) { CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); @@ -585,14 +594,11 @@ public List getAllPartitions() { builder.setPartitionName(partitionDescProto.getPartitionName()); builder.setPath(partitionDescProto.getPath()); - - // PARTITION_ID and TID is always necessary variables. In other CatalogStore excepting MemStore, - // all partitions would have PARTITION_ID and TID. But MemStore doesn't contain these variable values because - // it is implemented for test purpose. Thus, we need to set each variables to 0. - builder.setPartitionId(0); - builder.setTid(0); + builder.setPartitionId(partitionId); + builder.setTid(tableId); protos.add(builder.build()); + partitionId++; } } return protos; @@ -600,6 +606,8 @@ public List getAllPartitions() { public List getAllPartitionKeys() { List protos = new ArrayList(); + int partitionId = 0; + Set partitionTables = partitions.keySet(); for (String partitionTable : partitionTables) { Map entryMap = partitions.get(partitionTable); @@ -610,12 +618,13 @@ public List getAllPartitionKeys() { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(partitionKey.getColumnName()); builder.setPartitionValue(partitionKey.getPartitionValue()); - builder.setPartitionId(0); + builder.setPartitionId(partitionId); protos.add(builder.build()); } + + partitionId++; } } - return protos; } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 0919a3f8de..408aef32be 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -77,8 +77,7 @@ public final void testAlterTableSetProperty() throws Exception { public final void testAlterTableAddPartition() throws Exception { executeDDL("create_partitioned_table.sql", null); - String simpleTableName = "partitioned_table"; - String tableName = CatalogUtil.buildFQName(getCurrentDatabase(), simpleTableName); + String tableName = CatalogUtil.buildFQName("TestAlterTable", "partitioned_table"); assertTrue(catalog.existsTable(tableName)); TableDesc retrieved = catalog.getTableDesc(tableName); @@ -91,8 +90,7 @@ public final void testAlterTableAddPartition() throws Exception { executeDDL("alter_table_add_partition1.sql", null); executeDDL("alter_table_add_partition2.sql", null); - List partitions = catalog.getPartitions(getCurrentDatabase() - , simpleTableName); + List partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 1); assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2"); @@ -107,33 +105,10 @@ public final void testAlterTableAddPartition() throws Exception { assertTrue(fs.exists(partitionPath)); assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); - boolean existPartition = false; - List allPartitions = catalog.getAllPartitions(); - for (CatalogProtos.PartitionDescProto partition : allPartitions) { - if (partition.getPartitionName().equals("col3=1/col4=2") - && partition.getPath().equals(retrieved.getUri().toString() + "/col3=1/col4=2") - ){ - existPartition = true; - break; - } - } - assertTrue(existPartition); - - boolean existPartitionKey = false; - List tablePartitionKeys = catalog.getAllPartitionKeys(); - for (CatalogProtos.PartitionKeyProto partitionKey: tablePartitionKeys) { - if ((partitionKey.getColumnName().equals("col3") && partitionKey.getPartitionValue().equals("1") - || partitionKey.getColumnName().equals("col4") && partitionKey.getPartitionValue().equals("2"))) { - existPartitionKey = true; - break; - } - } - assertTrue(existPartitionKey); - executeDDL("alter_table_drop_partition1.sql", null); executeDDL("alter_table_drop_partition2.sql", null); - partitions = catalog.getPartitions(getCurrentDatabase(), simpleTableName); + partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 0); assertFalse(fs.exists(partitionPath)); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java index 207f64db67..7fa53448b4 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java @@ -18,9 +18,22 @@ package org.apache.tajo.master; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.util.TUtil; import org.junit.Test; +import java.sql.ResultSet; +import java.util.List; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + public class TestNonForwardQueryResultSystemScanner extends QueryTestCaseBase { @Test public void testGetNextRowsForAggregateFunction() throws Exception { @@ -38,4 +51,89 @@ public void testGetNextRowsForTable() throws Exception { public void testGetClusterDetails() throws Exception { assertQueryStr("SELECT TYPE FROM INFORMATION_SCHEMA.CLUSTER"); } + + @Test + public final void testGetInformationSchema() throws Exception { + executeDDL("create_partitioned_table.sql", null); + + String simpleTableName = "information_schema_test_table"; + String tableName = CatalogUtil.buildFQName(getCurrentDatabase(), simpleTableName); + assertTrue(catalog.existsTable(tableName)); + + TableDesc retrieved = catalog.getTableDesc(tableName); + + executeDDL("alter_table_add_partition1.sql", null); + executeDDL("alter_table_add_partition2.sql", null); + + List allDatabases = catalog.getAllDatabases(); + int dbId = -1; + for (CatalogProtos.DatabaseProto database : allDatabases) { + if (database.getName().equals(getCurrentDatabase())) { + dbId = database.getId(); + } + } + assertNotEquals(dbId, -1); + + int tableId = -1; + List allTables = catalog.getAllTables(); + for(CatalogProtos.TableDescriptorProto table : allTables) { + if (table.getDbId() == dbId && table.getName().equals(simpleTableName)) { + tableId = table.getTid(); + } + } + assertNotEquals(tableId, -1); + + List allPartitions = catalog.getAllPartitions(); + List resultPartitions = TUtil.newList(); + + int partitionId = 0; + for (CatalogProtos.PartitionDescProto partition : allPartitions) { + if (partition.getTid() == tableId + && partition.getPartitionName().equals("col3=1/col4=2") + && partition.getPath().equals(retrieved.getUri().toString() + "/col3=1/col4=2") + ){ + resultPartitions.add(partition); + partitionId = partition.getPartitionId(); + } + } + assertEquals(resultPartitions.size(), 1); + assertEquals(resultPartitions.get(0).getPartitionName(), "col3=1/col4=2"); + + List tablePartitionKeys = catalog.getAllPartitionKeys(); + List resultPartitionKeys = TUtil.newList(); + + for (CatalogProtos.PartitionKeyProto partitionKey: tablePartitionKeys) { + if (partitionKey.getPartitionId() == partitionId + && (partitionKey.getColumnName().equals("col3") && partitionKey.getPartitionValue().equals("1") + || partitionKey.getColumnName().equals("col4") && partitionKey.getPartitionValue().equals("2"))) { + resultPartitionKeys.add(partitionKey); + } + } + assertEquals(resultPartitionKeys.size(), 2); + assertEquals(resultPartitionKeys.get(0).getColumnName(), "col3"); + assertEquals(resultPartitionKeys.get(0).getPartitionValue(), "1"); + assertEquals(resultPartitionKeys.get(1).getColumnName(), "col4"); + assertEquals(resultPartitionKeys.get(1).getPartitionValue(), "2"); + + ResultSet resultSet = executeString("SELECT partition_name FROM INFORMATION_SCHEMA.PARTITIONS " + + " WHERE partition_id = " + partitionId); + + String actualResult = resultSetToString(resultSet); + String expectedResult = "partition_name\n" + + "-------------------------------\n" + + "col3=1/col4=2\n"; + assertEquals(expectedResult, actualResult); + + resultSet = executeString("SELECT column_name,partition_value FROM INFORMATION_SCHEMA.PARTITION_KEYS" + + " WHERE partition_id = " + partitionId); + + actualResult = resultSetToString(resultSet); + expectedResult = "column_name,partition_value\n" + + "-------------------------------\n" + + "col3,1\n" + + "col4,2\n"; + assertEquals(expectedResult, actualResult); + + executeDDL("drop_partitioned_table.sql", null); + } } diff --git a/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition1.sql b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition1.sql new file mode 100644 index 0000000000..9ce9697e7e --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition1.sql @@ -0,0 +1 @@ +ALTER TABLE information_schema_test_table ADD PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition2.sql b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition2.sql new file mode 100644 index 0000000000..4df56df587 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition2.sql @@ -0,0 +1 @@ +ALTER TABLE information_schema_test_table ADD IF NOT EXISTS PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/create_partitioned_table.sql b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/create_partitioned_table.sql new file mode 100644 index 0000000000..b6114b729c --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/create_partitioned_table.sql @@ -0,0 +1 @@ +create table information_schema_test_table (col1 int4, col2 int4) partition by column(col3 int4, col4 int4) \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/drop_partitioned_table.sql b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/drop_partitioned_table.sql new file mode 100644 index 0000000000..f0ae4cafb7 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/drop_partitioned_table.sql @@ -0,0 +1 @@ +drop table information_schema_test_table \ No newline at end of file From 1c4f86cba453e56e30085828be0a765748f52c20 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 26 Aug 2015 00:33:22 +0900 Subject: [PATCH 17/25] Remove unused codes --- .../tajo-catalog-common/src/main/proto/CatalogProtos.proto | 1 - .../test/java/org/apache/tajo/engine/query/TestAlterTable.java | 2 -- .../tajo/master/TestNonForwardQueryResultSystemScanner.java | 3 --- 3 files changed, 6 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 2233f5d154..b0fcccc44e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -246,7 +246,6 @@ message PartitionDescProto { message PartitionKeyProto { required string columnName = 1; - optional string parentColumnName = 2; optional string partitionValue = 3; optional int32 partition_id = 4; } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 408aef32be..8339ea71d2 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -25,7 +25,6 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.util.TUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -36,7 +35,6 @@ @Category(IntegrationTest.class) public class TestAlterTable extends QueryTestCaseBase { - @Test public final void testAlterTableName() throws Exception { List createdNames = executeDDL("table1_ddl.sql", "table1.tbl", "ABC"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java index 7fa53448b4..67eded9e38 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java @@ -18,8 +18,6 @@ package org.apache.tajo.master; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; @@ -32,7 +30,6 @@ import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; public class TestNonForwardQueryResultSystemScanner extends QueryTestCaseBase { @Test From fd8c6007e71bd7b897c772ef019195ad8b36b325 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 26 Aug 2015 01:12:56 +0900 Subject: [PATCH 18/25] Use tajo client APIs instead of catalog client APIs in test class. --- ...estNonForwardQueryResultSystemScanner.java | 77 ++++--------------- 1 file changed, 17 insertions(+), 60 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java index 67eded9e38..ab000d3940 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java @@ -53,73 +53,30 @@ public void testGetClusterDetails() throws Exception { public final void testGetInformationSchema() throws Exception { executeDDL("create_partitioned_table.sql", null); + ResultSet resultSet = null; + String actualResult, expectedResult; String simpleTableName = "information_schema_test_table"; - String tableName = CatalogUtil.buildFQName(getCurrentDatabase(), simpleTableName); - assertTrue(catalog.existsTable(tableName)); - - TableDesc retrieved = catalog.getTableDesc(tableName); executeDDL("alter_table_add_partition1.sql", null); executeDDL("alter_table_add_partition2.sql", null); - List allDatabases = catalog.getAllDatabases(); - int dbId = -1; - for (CatalogProtos.DatabaseProto database : allDatabases) { - if (database.getName().equals(getCurrentDatabase())) { - dbId = database.getId(); - } - } - assertNotEquals(dbId, -1); - - int tableId = -1; - List allTables = catalog.getAllTables(); - for(CatalogProtos.TableDescriptorProto table : allTables) { - if (table.getDbId() == dbId && table.getName().equals(simpleTableName)) { - tableId = table.getTid(); - } - } - assertNotEquals(tableId, -1); - - List allPartitions = catalog.getAllPartitions(); - List resultPartitions = TUtil.newList(); - - int partitionId = 0; - for (CatalogProtos.PartitionDescProto partition : allPartitions) { - if (partition.getTid() == tableId - && partition.getPartitionName().equals("col3=1/col4=2") - && partition.getPath().equals(retrieved.getUri().toString() + "/col3=1/col4=2") - ){ - resultPartitions.add(partition); - partitionId = partition.getPartitionId(); - } + resultSet = executeString("SELECT tid, table_name FROM INFORMATION_SCHEMA.TABLES " + + " WHERE TABLE_NAME = '" + simpleTableName + "'"); + int tid = -1; + while (resultSet.next()) { + tid = resultSet.getInt("tid"); } - assertEquals(resultPartitions.size(), 1); - assertEquals(resultPartitions.get(0).getPartitionName(), "col3=1/col4=2"); - - List tablePartitionKeys = catalog.getAllPartitionKeys(); - List resultPartitionKeys = TUtil.newList(); - - for (CatalogProtos.PartitionKeyProto partitionKey: tablePartitionKeys) { - if (partitionKey.getPartitionId() == partitionId - && (partitionKey.getColumnName().equals("col3") && partitionKey.getPartitionValue().equals("1") - || partitionKey.getColumnName().equals("col4") && partitionKey.getPartitionValue().equals("2"))) { - resultPartitionKeys.add(partitionKey); - } + resultSet.close(); + assertTrue(tid > -1); + + int partitionId = -1; + resultSet = executeString("SELECT partition_id, partition_name FROM INFORMATION_SCHEMA.PARTITIONS " + + " WHERE tid = " + tid); + while (resultSet.next()) { + partitionId = resultSet.getInt("partition_id"); } - assertEquals(resultPartitionKeys.size(), 2); - assertEquals(resultPartitionKeys.get(0).getColumnName(), "col3"); - assertEquals(resultPartitionKeys.get(0).getPartitionValue(), "1"); - assertEquals(resultPartitionKeys.get(1).getColumnName(), "col4"); - assertEquals(resultPartitionKeys.get(1).getPartitionValue(), "2"); - - ResultSet resultSet = executeString("SELECT partition_name FROM INFORMATION_SCHEMA.PARTITIONS " - + " WHERE partition_id = " + partitionId); - - String actualResult = resultSetToString(resultSet); - String expectedResult = "partition_name\n" + - "-------------------------------\n" + - "col3=1/col4=2\n"; - assertEquals(expectedResult, actualResult); + resultSet.close(); + assertTrue(partitionId > -1); resultSet = executeString("SELECT column_name,partition_value FROM INFORMATION_SCHEMA.PARTITION_KEYS" + " WHERE partition_id = " + partitionId); From 2092a2571e50707f2ebc760ef4e92305dba9392d Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 26 Aug 2015 01:50:44 +0900 Subject: [PATCH 19/25] Add more partitions when initializing in unit test cases. --- ...estNonForwardQueryResultSystemScanner.java | 43 ++++++++++++------- .../alter_table_add_partition1.sql | 1 - .../alter_table_add_partition2.sql | 1 - 3 files changed, 28 insertions(+), 17 deletions(-) delete mode 100644 tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition1.sql delete mode 100644 tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition2.sql diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java index ab000d3940..851809f47b 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java @@ -54,11 +54,13 @@ public final void testGetInformationSchema() throws Exception { executeDDL("create_partitioned_table.sql", null); ResultSet resultSet = null; - String actualResult, expectedResult; String simpleTableName = "information_schema_test_table"; + int totalCount = 1000; - executeDDL("alter_table_add_partition1.sql", null); - executeDDL("alter_table_add_partition2.sql", null); + for (int i = 0; i < totalCount; i++) { + executeString("ALTER TABLE information_schema_test_table ADD PARTITION (col3 = " + i + + ", col4 = " + (i + 1) + ")"); + } resultSet = executeString("SELECT tid, table_name FROM INFORMATION_SCHEMA.TABLES " + " WHERE TABLE_NAME = '" + simpleTableName + "'"); @@ -69,24 +71,35 @@ public final void testGetInformationSchema() throws Exception { resultSet.close(); assertTrue(tid > -1); - int partitionId = -1; - resultSet = executeString("SELECT partition_id, partition_name FROM INFORMATION_SCHEMA.PARTITIONS " + int resultPartitionCount = 0; + resultSet = executeString("SELECT count(*) FROM INFORMATION_SCHEMA.PARTITIONS " + " WHERE tid = " + tid); while (resultSet.next()) { - partitionId = resultSet.getInt("partition_id"); + resultPartitionCount = resultSet.getInt(1); } resultSet.close(); - assertTrue(partitionId > -1); + assertEquals(totalCount, resultPartitionCount); - resultSet = executeString("SELECT column_name,partition_value FROM INFORMATION_SCHEMA.PARTITION_KEYS" + - " WHERE partition_id = " + partitionId); + // Setting select statement for getting partition keys. + String selectPartitionKeys = "SELECT count(*) FROM INFORMATION_SCHEMA.PARTITION_KEYS WHERE partition_id IN ("; + resultSet = executeString("SELECT partition_id FROM INFORMATION_SCHEMA.PARTITIONS "); + int i = 0; + while (resultSet.next()) { + if (i > 0) { + selectPartitionKeys += ","; + } + selectPartitionKeys += resultSet.getInt(1); + i++; + } + selectPartitionKeys += ")"; + resultSet.close(); - actualResult = resultSetToString(resultSet); - expectedResult = "column_name,partition_value\n" + - "-------------------------------\n" + - "col3,1\n" + - "col4,2\n"; - assertEquals(expectedResult, actualResult); + resultSet = executeString(selectPartitionKeys); + int resultPartitionKeyCount = 0; + while (resultSet.next()) { + resultPartitionKeyCount = resultSet.getInt(1); + } + assertEquals(totalCount * 2, resultPartitionKeyCount); executeDDL("drop_partitioned_table.sql", null); } diff --git a/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition1.sql b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition1.sql deleted file mode 100644 index 9ce9697e7e..0000000000 --- a/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition1.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE information_schema_test_table ADD PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition2.sql b/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition2.sql deleted file mode 100644 index 4df56df587..0000000000 --- a/tajo-core-tests/src/test/resources/queries/TestNonForwardQueryResultSystemScanner/alter_table_add_partition2.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE information_schema_test_table ADD IF NOT EXISTS PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file From e0e86de4acc06f249c52c550d7fd12db87b02742 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 27 Aug 2015 11:27:42 +0900 Subject: [PATCH 20/25] Add GetPartitionDescProto and GetPartitionKeyProto --- .../tajo/catalog/AbstractCatalogClient.java | 8 ++-- .../src/main/proto/CatalogProtocol.proto | 6 +-- .../apache/tajo/catalog/CatalogService.java | 8 ++-- .../src/main/proto/CatalogProtos.proto | 17 +++++-- .../tajo/catalog/store/HiveCatalogStore.java | 23 +++++----- .../catalog/store/TestHiveCatalogStore.java | 8 ++-- .../apache/tajo/catalog/CatalogServer.java | 6 +-- .../tajo/catalog/store/AbstractDBStore.java | 35 +++++++------- .../tajo/catalog/store/CatalogStore.java | 10 ++-- .../apache/tajo/catalog/store/MemStore.java | 46 +++++++++---------- .../org/apache/tajo/catalog/TestCatalog.java | 9 ++-- .../tajo/engine/query/TestAlterTable.java | 3 +- .../engine/query/TestTablePartitions.java | 10 ++-- ...estNonForwardQueryResultSystemScanner.java | 5 -- .../apache/tajo/master/exec/DDLExecutor.java | 6 +-- .../NonForwardQueryResultSystemScanner.java | 8 ++-- 16 files changed, 105 insertions(+), 103 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 3901bc878f..0283cbc3cf 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -401,7 +401,7 @@ public final boolean existPartitionMethod(final String databaseName, final Strin } @Override - public final PartitionDescProto getPartition(final String databaseName, final String tableName, + public final GetPartitionDescProto getPartition(final String databaseName, final String tableName, final String partitionName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionException, UndefinedPartitionMethodException { @@ -428,7 +428,7 @@ public final PartitionDescProto getPartition(final String databaseName, final St } @Override - public final List getPartitions(final String databaseName, final String tableName) { + public final List getPartitions(final String databaseName, final String tableName) { try { final BlockingInterface stub = getStub(); final PartitionIdentifierProto request = PartitionIdentifierProto.newBuilder() @@ -446,7 +446,7 @@ public final List getPartitions(final String databaseName, f } @Override - public List getAllPartitions() { + public List getAllPartitions() { try { final BlockingInterface stub = getStub(); final GetPartitionsResponse response = stub.getAllPartitions(null, ProtoUtil.NULL_PROTO); @@ -491,7 +491,7 @@ public void addPartitions(String databaseName, String tableName, List getAllPartitionKeys() { + public List getAllPartitionKeys() { try { final BlockingInterface stub = getStub(); final GetPartitionKeysResponse response = stub.getAllPartitionKeys(null, ProtoUtil.NULL_PROTO); diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index e57fddbc67..7af7b363d9 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -66,7 +66,7 @@ message GetPartitionMethodResponse { message GetPartitionDescResponse { required ReturnState state = 1; - optional PartitionDescProto partition = 2; + optional GetPartitionDescProto partition = 2; } message GetIndexeDescResponse { @@ -81,12 +81,12 @@ message GetIndexByColumnNamesRequest { message GetPartitionsResponse { required ReturnState state = 1; - repeated PartitionDescProto partition = 2; + repeated GetPartitionDescProto partition = 2; } message GetPartitionKeysResponse { required ReturnState state = 1; - repeated PartitionKeyProto partitionKey = 2; + repeated GetPartitionKeyProto partitionKey = 2; } service CatalogProtocolService { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index 8546bb8eb0..8cff2cbd8a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -158,18 +158,18 @@ void alterTablespace(AlterTablespaceProto alterTablespace) boolean existPartitionMethod(String databaseName, String tableName) throws UndefinedTableException, UndefinedDatabaseException; - PartitionDescProto getPartition(String databaseName, String tableName, String partitionName) + GetPartitionDescProto getPartition(String databaseName, String tableName, String partitionName) throws UndefinedPartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException, UndefinedTableException; - List getPartitions(String databaseName, String tableName); + List getPartitions(String databaseName, String tableName); - List getAllPartitions(); + List getAllPartitions(); void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws UndefinedTableException, DuplicatePartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException; - List getAllPartitionKeys(); + List getAllPartitionKeys(); boolean createIndex(IndexDesc index); diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index b0fcccc44e..0e098ec9ab 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -240,14 +240,25 @@ message PartitionDescProto { required string partitionName = 1; repeated PartitionKeyProto partitionKeys = 2; optional string path = 3; - optional int32 partition_id = 4; - optional int32 tid = 5; } message PartitionKeyProto { required string columnName = 1; optional string partitionValue = 3; - optional int32 partition_id = 4; +} + +message GetPartitionDescProto { + required string partitionName = 1; + repeated GetPartitionKeyProto partitionKeys = 2; + optional string path = 3; + required int32 partition_id = 4; + required int32 tid = 5; +} + +message GetPartitionKeyProto { + required string columnName = 1; + optional string partitionValue = 3; + required int32 partition_id = 4; } message PartitionIdentifierProto { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index d9a7b172b7..693c0714fc 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -588,7 +588,7 @@ public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescPro final String databaseName = split[0]; final String tableName = split[1]; String partitionName = null; - CatalogProtos.PartitionDescProto partitionDesc = null; + GetPartitionDescProto partitionDesc = null; switch (alterTableDescProto.getAlterTableType()) { case RENAME_TABLE: @@ -732,15 +732,14 @@ private void addPartition(String databaseName, String tableName, CatalogProtos.P } } - private void dropPartition(String databaseName, String tableName, CatalogProtos.PartitionDescProto - partitionDescProto) { + private void dropPartition(String databaseName, String tableName, GetPartitionDescProto partitionDescProto) { HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; try { client = clientPool.getClient(); List values = Lists.newArrayList(); - for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { + for(GetPartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { values.add(keyProto.getPartitionValue()); } client.getHiveClient().dropPartition(databaseName, tableName, values, true); @@ -838,23 +837,23 @@ public boolean existPartitionMethod(String databaseName, String tableName) throw } @Override - public List getPartitions(String databaseName, + public List getPartitions(String databaseName, String tableName) { throw new UnsupportedOperationException(); } @Override - public CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, + public GetPartitionDescProto getPartition(String databaseName, String tableName, String partitionName) { HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; - CatalogProtos.PartitionDescProto.Builder builder = null; + GetPartitionDescProto.Builder builder = null; try { client = clientPool.getClient(); Partition partition = client.getHiveClient().getPartition(databaseName, tableName, partitionName); - builder = CatalogProtos.PartitionDescProto.newBuilder(); + builder = GetPartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); builder.setPath(partition.getSd().getLocation()); @@ -862,7 +861,7 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String for (int i = 0; i < partition.getValues().size(); i++) { String value = partition.getValues().get(i); - CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder(); + GetPartitionKeyProto.Builder keyBuilder = GetPartitionKeyProto.newBuilder(); String columnName = partitionNames[i].split("=")[0]; keyBuilder.setColumnName(columnName); @@ -1005,12 +1004,12 @@ public List getAllIndexes() { } @Override - public List getAllPartitions() { + public List getAllPartitions() { throw new UnsupportedOperationException(); } @Override - public List getAllPartitionKeys() { + public List getAllPartitionKeys() { throw new UnsupportedOperationException(); } @@ -1019,7 +1018,7 @@ public void addPartitions(String databaseName, String tableName, List addPartitions = TUtil.newList(); - CatalogProtos.PartitionDescProto existingPartition = null; + GetPartitionDescProto existingPartition = null; try { client = clientPool.getClient(); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index b3af179421..e1dcf9804b 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -29,6 +29,8 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; +import org.apache.tajo.catalog.proto.CatalogProtos.GetPartitionDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.GetPartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageConstants; @@ -266,7 +268,7 @@ public void testAddTableByPartition() throws Exception { testDropPartition(NATION, "n_nationkey=10/n_date=20150101"); testDropPartition(NATION, "n_nationkey=20/n_date=20150102"); - CatalogProtos.PartitionDescProto partition = store.getPartition(DB_NAME, NATION, "n_nationkey=10/n_date=20150101"); + GetPartitionDescProto partition = store.getPartition(DB_NAME, NATION, "n_nationkey=10/n_date=20150101"); assertNull(partition); partition = store.getPartition(DB_NAME, NATION, "n_nationkey=20/n_date=20150102"); @@ -303,14 +305,14 @@ private void testAddPartition(URI uri, String tableName, String partitionName) t store.alterTable(alterTableDesc.getProto()); - CatalogProtos.PartitionDescProto resultDesc = store.getPartition(DB_NAME, NATION, partitionName); + GetPartitionDescProto resultDesc = store.getPartition(DB_NAME, NATION, partitionName); assertNotNull(resultDesc); assertEquals(resultDesc.getPartitionName(), partitionName); assertEquals(resultDesc.getPath(), uri.toString() + "/" + partitionName); assertEquals(resultDesc.getPartitionKeysCount(), 2); for (int i = 0; i < resultDesc.getPartitionKeysCount(); i++) { - CatalogProtos.PartitionKeyProto keyProto = resultDesc.getPartitionKeys(i); + GetPartitionKeyProto keyProto = resultDesc.getPartitionKeys(i); String[] eachName = partitionNames[i].split("="); assertEquals(keyProto.getPartitionValue(), eachName[1]); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index f093c8044c..9921522e8f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -928,7 +928,7 @@ public GetPartitionDescResponse getPartitionByPartitionName(RpcController contro rlock.lock(); try { - PartitionDescProto partitionDesc = store.getPartition(dbName, tbName, partitionName); + GetPartitionDescProto partitionDesc = store.getPartition(dbName, tbName, partitionName); if (partitionDesc != null) { return GetPartitionDescResponse.newBuilder() .setState(OK) @@ -976,10 +976,10 @@ public GetPartitionsResponse getPartitionsByTableName(RpcController controller, rlock.lock(); try { - List partitions = store.getPartitions(dbName, tbName); + List partitions = store.getPartitions(dbName, tbName); GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder(); - for (PartitionDescProto partition : partitions) { + for (GetPartitionDescProto partition : partitions) { builder.addPartition(partition); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 415c91addb..048b80cf68 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1001,7 +1001,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) final String databaseName = splitted[0]; final String tableName = splitted[1]; String partitionName = null; - CatalogProtos.PartitionDescProto partitionDesc = null; + GetPartitionDescProto partitionDesc = null; int databaseId = getDatabaseId(databaseName); int tableId = getTableId(databaseId, databaseName, tableName); @@ -2015,7 +2015,7 @@ private void ensurePartitionTable(String tbName, int tableId) } @Override - public CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, + public GetPartitionDescProto getPartition(String databaseName, String tableName, String partitionName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, UndefinedPartitionException { @@ -2027,7 +2027,7 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; - PartitionDescProto.Builder builder = null; + GetPartitionDescProto.Builder builder = null; try { String sql = "SELECT PATH, " + COL_PARTITIONS_PK + " FROM " + TB_PARTTIONS + @@ -2044,7 +2044,7 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String res = pstmt.executeQuery(); if (res.next()) { - builder = PartitionDescProto.newBuilder(); + builder = GetPartitionDescProto.newBuilder(); builder.setPartitionId(res.getInt(COL_PARTITIONS_PK)); builder.setPath(res.getString("PATH")); builder.setPartitionName(partitionName); @@ -2060,7 +2060,7 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String return builder.build(); } - private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) { + private void setPartitionKeys(int pid, GetPartitionDescProto.Builder partitionDesc) { Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; @@ -2075,7 +2075,7 @@ private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) res = pstmt.executeQuery(); while (res.next()) { - PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + GetPartitionKeyProto.Builder builder = GetPartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); partitionDesc.addPartitionKeys(builder); @@ -2088,14 +2088,14 @@ private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) } @Override - public List getPartitions(String databaseName, String tableName) + public List getPartitions(String databaseName, String tableName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; - PartitionDescProto.Builder builder = null; - List partitions = new ArrayList(); + GetPartitionDescProto.Builder builder = null; + List partitions = new ArrayList(); final int databaseId = getDatabaseId(databaseName); final int tableId = getTableId(databaseId, databaseName, tableName); @@ -2115,7 +2115,7 @@ public List getPartitions(String databaseName, String tableN res = pstmt.executeQuery(); while (res.next()) { - builder = PartitionDescProto.newBuilder(); + builder = GetPartitionDescProto.newBuilder(); builder.setPath(res.getString("PATH")); builder.setPartitionName(res.getString("PARTITION_NAME")); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); @@ -2130,12 +2130,12 @@ public List getPartitions(String databaseName, String tableN } @Override - public List getAllPartitions() { + public List getAllPartitions() { Connection conn = null; Statement stmt = null; ResultSet resultSet = null; - List partitions = new ArrayList(); + List partitions = new ArrayList(); try { String sql = "SELECT A." + COL_PARTITIONS_PK + ", A." + COL_TABLES_PK + ", A.PARTITION_NAME, A.PATH " @@ -2145,7 +2145,7 @@ public List getAllPartitions() { stmt = conn.createStatement(); resultSet = stmt.executeQuery(sql); while (resultSet.next()) { - PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); + GetPartitionDescProto.Builder builder = GetPartitionDescProto.newBuilder(); builder.setPartitionId(resultSet.getInt(COL_PARTITIONS_PK)); builder.setTid(resultSet.getInt(COL_TABLES_PK)); @@ -2183,7 +2183,7 @@ public void addPartitions(String databaseName, String tableName, List getAllPartitionKeys() { + public List getAllPartitionKeys() { Connection conn = null; Statement stmt = null; ResultSet resultSet = null; - List partitions = new ArrayList(); + List partitions = new ArrayList(); try { String sql = " SELECT A." + COL_PARTITIONS_PK + ", A.COLUMN_NAME, A.PARTITION_VALUE " + @@ -2293,12 +2293,11 @@ public List getAllPartitionKeys() { stmt = conn.createStatement(); resultSet = stmt.executeQuery(sql); while (resultSet.next()) { - PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + GetPartitionKeyProto.Builder builder = GetPartitionKeyProto.newBuilder(); builder.setPartitionId(resultSet.getInt(COL_PARTITIONS_PK)); builder.setColumnName(resultSet.getString("COLUMN_NAME")); builder.setPartitionValue(resultSet.getString("PARTITION_VALUE")); - partitions.add(builder.build()); } } catch (SQLException se) { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 1b7c8ae367..393495d546 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -18,14 +18,12 @@ package org.apache.tajo.catalog.store; -import com.google.protobuf.InvalidProtocolBufferException; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.exception.*; import java.io.Closeable; -import java.sql.SQLException; import java.util.Collection; import java.util.List; @@ -97,17 +95,17 @@ boolean existPartitionMethod(String databaseName, String tableName) throws Undef * @return * @throws TajoException */ - List getPartitions(String databaseName, String tableName) throws + List getPartitions(String databaseName, String tableName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException; - CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, + GetPartitionDescProto getPartition(String databaseName, String tableName, String partitionName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionException, UndefinedPartitionMethodException; - List getAllPartitions(); + List getAllPartitions(); - List getAllPartitionKeys(); + List getAllPartitionKeys(); void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws UndefinedDatabaseException, diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index ff9377aaf1..052ee8cc67 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -48,7 +48,7 @@ public class MemStore implements CatalogStore { private final Map functions = Maps.newHashMap(); private final Map> indexes = Maps.newHashMap(); private final Map> indexesByColumn = Maps.newHashMap(); - private final Map> partitions = Maps.newHashMap(); + private final Map> partitions = Maps.newHashMap(); public MemStore(Configuration conf) { } @@ -337,21 +337,21 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) } } - private void addPartition(CatalogProtos.PartitionDescProto partitionDesc, String tableName, String partitionName) { - CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); + private void addPartition(PartitionDescProto partitionDesc, String tableName, String partitionName) { + GetPartitionDescProto.Builder builder = GetPartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); builder.setPath(partitionDesc.getPath()); if (partitionDesc.getPartitionKeysCount() > 0) { - for (CatalogProtos.PartitionKeyProto eachKey : partitionDesc.getPartitionKeysList()) { - CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder(); + for (PartitionKeyProto eachKey : partitionDesc.getPartitionKeysList()) { + GetPartitionKeyProto.Builder keyBuilder = GetPartitionKeyProto.newBuilder(); keyBuilder.setColumnName(eachKey.getColumnName()); keyBuilder.setPartitionValue(eachKey.getPartitionValue()); builder.addPartitionKeys(keyBuilder.build()); } } - Map protoMap = null; + Map protoMap = null; if (!partitions.containsKey(tableName)) { protoMap = Maps.newHashMap(); } else { @@ -552,11 +552,11 @@ public boolean existPartitionMethod(String databaseName, String tableName) } @Override - public List getPartitions(String databaseName, String tableName) { - List protos = new ArrayList(); + public List getPartitions(String databaseName, String tableName) { + List protos = TUtil.newList(); if (partitions.containsKey(tableName)) { - for (CatalogProtos.PartitionDescProto proto : partitions.get(tableName).values()) { + for (GetPartitionDescProto proto : partitions.get(tableName).values()) { protos.add(proto); } } @@ -564,7 +564,7 @@ public List getPartitions(String databaseName, } @Override - public CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, + public GetPartitionDescProto getPartition(String databaseName, String tableName, String partitionName) throws UndefinedPartitionException { if (partitions.containsKey(tableName) && partitions.get(tableName).containsKey(partitionName)) { return partitions.get(tableName).get(partitionName); @@ -573,10 +573,10 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String } } - public List getAllPartitions() { + public List getAllPartitions() { int tableId = 0, partitionId = 0; List tables = getAllTables(); - List protos = new ArrayList(); + List protos = TUtil.newList(); Set partitionTables = partitions.keySet(); for (String partitionTable : partitionTables) { @@ -586,11 +586,11 @@ public List getAllPartitions() { } } - Map entryMap = partitions.get(partitionTable); - for (Map.Entry proto : entryMap.entrySet()) { - CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); + Map entryMap = partitions.get(partitionTable); + for (Map.Entry proto : entryMap.entrySet()) { + GetPartitionDescProto partitionDescProto = proto.getValue(); - PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); + GetPartitionDescProto.Builder builder = GetPartitionDescProto.newBuilder(); builder.setPartitionName(partitionDescProto.getPartitionName()); builder.setPath(partitionDescProto.getPath()); @@ -604,18 +604,18 @@ public List getAllPartitions() { return protos; } - public List getAllPartitionKeys() { - List protos = new ArrayList(); + public List getAllPartitionKeys() { + List protos = TUtil.newList(); int partitionId = 0; Set partitionTables = partitions.keySet(); for (String partitionTable : partitionTables) { - Map entryMap = partitions.get(partitionTable); - for (Map.Entry proto : entryMap.entrySet()) { - CatalogProtos.PartitionDescProto partitionDescProto = proto.getValue(); + Map entryMap = partitions.get(partitionTable); + for (Map.Entry proto : entryMap.entrySet()) { + GetPartitionDescProto partitionDescProto = proto.getValue(); - for (PartitionKeyProto partitionKey : partitionDescProto.getPartitionKeysList()) { - PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + for (GetPartitionKeyProto partitionKey : partitionDescProto.getPartitionKeysList()) { + GetPartitionKeyProto.Builder builder = GetPartitionKeyProto.newBuilder(); builder.setColumnName(partitionKey.getColumnName()); builder.setPartitionValue(partitionKey.getPartitionValue()); builder.setPartitionId(partitionId); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 1d628f2968..1c0ac9f2c5 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -25,9 +25,7 @@ import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; -import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; -import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; +import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.catalog.store.*; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; @@ -50,7 +48,6 @@ import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.catalog.CatalogConstants.CATALOG_URI; -import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation; import static org.junit.Assert.*; @@ -930,7 +927,7 @@ public final void testAddAndDeleteTablePartitionByColumn() throws Exception { testAddPartition(tableName, "id=10/name=aaa"); testAddPartition(tableName, "id=20/name=bbb"); - List partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable"); + List partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable"); assertNotNull(partitions); assertEquals(partitions.size(), 2); @@ -974,7 +971,7 @@ private void testAddPartition(String tableName, String partitionName) throws Exc catalog.alterTable(alterTableDesc); - CatalogProtos.PartitionDescProto resultDesc = catalog.getPartition(DEFAULT_DATABASE_NAME, + GetPartitionDescProto resultDesc = catalog.getPartition(DEFAULT_DATABASE_NAME, "addedtable", partitionName); assertNotNull(resultDesc); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 8339ea71d2..0772f08cbc 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -25,6 +25,7 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -88,7 +89,7 @@ public final void testAlterTableAddPartition() throws Exception { executeDDL("alter_table_add_partition1.sql", null); executeDDL("alter_table_add_partition2.sql", null); - List partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + List partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 1); assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index 952e26a38b..21c694a96e 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -29,7 +29,7 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.exception.ReturnStateUtil; -import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.DataChannel; @@ -1268,7 +1268,7 @@ private void verifyPartitionDirectoryFromCatalog(String databaseName, String tab ResultSet res = executeString(query.toString()); StringBuilder partitionName = new StringBuilder(); - PartitionDescProto partitionDescProto = null; + GetPartitionDescProto partitionDescProto = null; // Check whether that partition's directory exist or doesn't exist. while(res.next()) { @@ -1324,12 +1324,12 @@ public final void testDuplicatedPartitions() throws Exception { // If duplicated partitions had been removed, partitions just will contain 'KEY=N' partition and 'KEY=R' // partition. In previous Query and Stage, duplicated partitions were not deleted because they had been in List. // If you want to verify duplicated partitions, you need to use List instead of Set with DerbyStore. - List partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, tableName); + List partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, tableName); assertEquals(2, partitions.size()); - PartitionDescProto firstPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=N"); + GetPartitionDescProto firstPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=N"); assertNotNull(firstPartition); - PartitionDescProto secondPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=R"); + GetPartitionDescProto secondPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=R"); assertNotNull(secondPartition); } finally { executeString("DROP TABLE lineitem2 PURGE"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java index 851809f47b..e998fcbf3e 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java @@ -19,14 +19,9 @@ package org.apache.tajo.master; import org.apache.tajo.QueryTestCaseBase; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.util.TUtil; import org.junit.Test; import java.sql.ResultSet; -import java.util.List; import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 09bef448de..7176d1f9e6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -27,9 +27,9 @@ import org.apache.tajo.algebra.AlterTablespaceSetType; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; +import org.apache.tajo.catalog.proto.CatalogProtos.GetPartitionDescProto; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.*; @@ -422,7 +422,7 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer Path partitionPath = null; TableDesc desc = null; Pair, String> pair = null; - CatalogProtos.PartitionDescProto partitionDescProto = null; + GetPartitionDescProto partitionDescProto = null; if (alterTable.getAlterTableOpType() == AlterTableOpType.RENAME_TABLE || alterTable.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION @@ -554,7 +554,7 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer } } - private void deletePartitionPath(CatalogProtos.PartitionDescProto partitionDescProto) throws IOException { + private void deletePartitionPath(GetPartitionDescProto partitionDescProto) throws IOException { Path partitionPath = new Path(partitionDescProto.getPath()); FileSystem fs = partitionPath.getFileSystem(context.getConf()); if (fs.exists(partitionPath)) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 11f68197f6..5fc281da8d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -385,12 +385,12 @@ private List getAllTableStats(Schema outSchema) { } private List getAllPartitions(Schema outSchema) { - List partitionList = masterContext.getCatalog().getAllPartitions(); + List partitionList = masterContext.getCatalog().getAllPartitions(); List tuples = new ArrayList(partitionList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; - for (PartitionDescProto partition: partitionList) { + for (GetPartitionDescProto partition: partitionList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { @@ -418,12 +418,12 @@ private List getAllPartitions(Schema outSchema) { } private List getAllPartitionKeys(Schema outSchema) { - List partitionKeyList = masterContext.getCatalog().getAllPartitionKeys(); + List partitionKeyList = masterContext.getCatalog().getAllPartitionKeys(); List tuples = new ArrayList(partitionKeyList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; - for (PartitionKeyProto partitionKey: partitionKeyList) { + for (GetPartitionKeyProto partitionKey: partitionKeyList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { From 4fcb26e8b923c8d3b5390a181b220912c6f6213a Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 27 Aug 2015 11:55:05 +0900 Subject: [PATCH 21/25] Fix unit test bugs --- .../java/org/apache/tajo/catalog/store/AbstractDBStore.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 048b80cf68..26ca7c63a2 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2045,6 +2045,7 @@ public GetPartitionDescProto getPartition(String databaseName, String tableName, if (res.next()) { builder = GetPartitionDescProto.newBuilder(); + builder.setTid(tableId); builder.setPartitionId(res.getInt(COL_PARTITIONS_PK)); builder.setPath(res.getString("PATH")); builder.setPartitionName(partitionName); @@ -2078,6 +2079,7 @@ private void setPartitionKeys(int pid, GetPartitionDescProto.Builder partitionDe GetPartitionKeyProto.Builder builder = GetPartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); + builder.setPartitionId(pid); partitionDesc.addPartitionKeys(builder); } } catch (SQLException se) { @@ -2116,6 +2118,8 @@ public List getPartitions(String databaseName, String tab while (res.next()) { builder = GetPartitionDescProto.newBuilder(); + builder.setTid(tableId); + builder.setPartitionId(res.getInt(COL_PARTITIONS_PK)); builder.setPath(res.getString("PATH")); builder.setPartitionName(res.getString("PARTITION_NAME")); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); From 97abd71d94651b01d293f31f7dbbf3fc10b82a3f Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 27 Aug 2015 12:02:02 +0900 Subject: [PATCH 22/25] Fix HiveCatalogStore unit test bug --- .../java/org/apache/tajo/catalog/store/HiveCatalogStore.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 693c0714fc..5e3d9972d1 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -856,6 +856,9 @@ public GetPartitionDescProto getPartition(String databaseName, String tableName, builder = GetPartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); builder.setPath(partition.getSd().getLocation()); + // These are dummy values because there is no way to get partition id and table id in HiveMetaStore. + builder.setPartitionId(0); + builder.setTid(0); String[] partitionNames = partitionName.split("/"); @@ -866,6 +869,8 @@ public GetPartitionDescProto getPartition(String databaseName, String tableName, String columnName = partitionNames[i].split("=")[0]; keyBuilder.setColumnName(columnName); keyBuilder.setPartitionValue(value); + // This is dummy value because there is no way to get partition id in HiveMetaStore. + keyBuilder.setPartitionId(0); builder.addPartitionKeys(keyBuilder); } } catch (NoSuchObjectException e) { From c206460b0744782ffcbbc6d3d983f2e98a46eee6 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 27 Aug 2015 12:37:40 +0900 Subject: [PATCH 23/25] Add table id and partition id to MemStore --- .../apache/tajo/catalog/store/MemStore.java | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 052ee8cc67..ed842e1b5e 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -31,6 +31,7 @@ import org.apache.tajo.exception.*; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import java.io.IOException; @@ -338,29 +339,54 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) } private void addPartition(PartitionDescProto partitionDesc, String tableName, String partitionName) { + + Map protoMap = null; + if (!partitions.containsKey(tableName)) { + protoMap = Maps.newHashMap(); + } else { + protoMap = partitions.get(tableName); + } + GetPartitionDescProto.Builder builder = GetPartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); builder.setPath(partitionDesc.getPath()); + builder.setPartitionId(protoMap.size() + 1); + builder.setTid(getTableId(tableName)); if (partitionDesc.getPartitionKeysCount() > 0) { for (PartitionKeyProto eachKey : partitionDesc.getPartitionKeysList()) { GetPartitionKeyProto.Builder keyBuilder = GetPartitionKeyProto.newBuilder(); keyBuilder.setColumnName(eachKey.getColumnName()); keyBuilder.setPartitionValue(eachKey.getPartitionValue()); + keyBuilder.setPartitionId(protoMap.size()+1); builder.addPartitionKeys(keyBuilder.build()); } } - Map protoMap = null; - if (!partitions.containsKey(tableName)) { - protoMap = Maps.newHashMap(); - } else { - protoMap = partitions.get(tableName); - } protoMap.put(partitionName, builder.build()); partitions.put(tableName, protoMap); } + private int getTableId(String tableName) { + int tableId = 0, retValue = 0; + + for (String databaseName: databases.keySet()) { + Map tables = databases.get(databaseName); + List tableNameList = TUtil.newList(tables.keySet()); + Collections.sort(tableNameList); + + for (String eachTableName: tableNameList) { + if (eachTableName.equals(tableName)) { + retValue = tableId; + break; + } + tableId++; + } + } + + return retValue; + } + private void dropPartition(String databaseName, String tableName, String partitionName) throws UndefinedPartitionException { if(!partitions.containsKey(tableName)) { @@ -574,18 +600,10 @@ public GetPartitionDescProto getPartition(String databaseName, String tableName, } public List getAllPartitions() { - int tableId = 0, partitionId = 0; - List tables = getAllTables(); List protos = TUtil.newList(); Set partitionTables = partitions.keySet(); for (String partitionTable : partitionTables) { - for (TableDescriptorProto table : tables) { - if (table.getName().equals(partitionTable)) { - tableId = table.getTid(); - } - } - Map entryMap = partitions.get(partitionTable); for (Map.Entry proto : entryMap.entrySet()) { GetPartitionDescProto partitionDescProto = proto.getValue(); @@ -594,11 +612,10 @@ public List getAllPartitions() { builder.setPartitionName(partitionDescProto.getPartitionName()); builder.setPath(partitionDescProto.getPath()); - builder.setPartitionId(partitionId); - builder.setTid(tableId); + builder.setPartitionId(partitionDescProto.getPartitionId()); + builder.setTid(partitionDescProto.getTid()); protos.add(builder.build()); - partitionId++; } } return protos; @@ -606,7 +623,6 @@ public List getAllPartitions() { public List getAllPartitionKeys() { List protos = TUtil.newList(); - int partitionId = 0; Set partitionTables = partitions.keySet(); for (String partitionTable : partitionTables) { @@ -618,11 +634,9 @@ public List getAllPartitionKeys() { GetPartitionKeyProto.Builder builder = GetPartitionKeyProto.newBuilder(); builder.setColumnName(partitionKey.getColumnName()); builder.setPartitionValue(partitionKey.getPartitionValue()); - builder.setPartitionId(partitionId); + builder.setPartitionId(partitionKey.getPartitionId()); protos.add(builder.build()); } - - partitionId++; } } return protos; From 33874c87dc0e71102a3a27c2dfcd810d1bb57a45 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 4 Sep 2015 15:07:58 +0900 Subject: [PATCH 24/25] Remove unnecessary updates --- .../tajo/catalog/AbstractCatalogClient.java | 14 +++--- .../src/main/proto/CatalogProtocol.proto | 17 ++++--- .../apache/tajo/catalog/CatalogService.java | 8 +-- .../src/main/proto/CatalogProtos.proto | 30 +++++------ .../tajo/catalog/store/HiveCatalogStore.java | 28 +++++------ .../catalog/store/TestHiveCatalogStore.java | 8 ++- .../apache/tajo/catalog/CatalogServer.java | 22 ++++---- .../tajo/catalog/store/AbstractDBStore.java | 50 +++++++++---------- .../tajo/catalog/store/CatalogStore.java | 8 +-- .../org/apache/tajo/catalog/TestCatalog.java | 10 ++-- .../tajo/engine/query/TestAlterTable.java | 3 +- .../engine/query/TestTablePartitions.java | 10 ++-- .../apache/tajo/master/exec/DDLExecutor.java | 6 +-- .../NonForwardQueryResultSystemScanner.java | 8 +-- 14 files changed, 111 insertions(+), 111 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 0283cbc3cf..2860234ca4 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -401,7 +401,7 @@ public final boolean existPartitionMethod(final String databaseName, final Strin } @Override - public final GetPartitionDescProto getPartition(final String databaseName, final String tableName, + public final PartitionDescProto getPartition(final String databaseName, final String tableName, final String partitionName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionException, UndefinedPartitionMethodException { @@ -428,7 +428,7 @@ public final GetPartitionDescProto getPartition(final String databaseName, final } @Override - public final List getPartitions(final String databaseName, final String tableName) { + public final List getPartitions(final String databaseName, final String tableName) { try { final BlockingInterface stub = getStub(); final PartitionIdentifierProto request = PartitionIdentifierProto.newBuilder() @@ -446,13 +446,13 @@ public final List getPartitions(final String databaseName } @Override - public List getAllPartitions() { + public List getAllPartitions() { try { final BlockingInterface stub = getStub(); - final GetPartitionsResponse response = stub.getAllPartitions(null, ProtoUtil.NULL_PROTO); + final GetTablePartitionsResponse response = stub.getAllPartitions(null, ProtoUtil.NULL_PROTO); ensureOk(response.getState()); - return response.getPartitionList(); + return response.getPartList(); } catch (ServiceException e) { throw new RuntimeException(e); @@ -491,10 +491,10 @@ public void addPartitions(String databaseName, String tableName, List getAllPartitionKeys() { + public List getAllPartitionKeys() { try { final BlockingInterface stub = getStub(); - final GetPartitionKeysResponse response = stub.getAllPartitionKeys(null, ProtoUtil.NULL_PROTO); + final GetTablePartitionKeysResponse response = stub.getAllPartitionKeys(null, ProtoUtil.NULL_PROTO); ensureOk(response.getState()); return response.getPartitionKeyList(); diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index 7af7b363d9..f9db20c426 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -66,7 +66,7 @@ message GetPartitionMethodResponse { message GetPartitionDescResponse { required ReturnState state = 1; - optional GetPartitionDescProto partition = 2; + optional PartitionDescProto partition = 2; } message GetIndexeDescResponse { @@ -81,12 +81,17 @@ message GetIndexByColumnNamesRequest { message GetPartitionsResponse { required ReturnState state = 1; - repeated GetPartitionDescProto partition = 2; + repeated PartitionDescProto partition = 2; } -message GetPartitionKeysResponse { +message GetTablePartitionsResponse { required ReturnState state = 1; - repeated GetPartitionKeyProto partitionKey = 2; + repeated TablePartitionProto part = 2; +} + +message GetTablePartitionKeysResponse { + required ReturnState state = 1; + repeated TablePartitionKeyProto partitionKey = 2; } service CatalogProtocolService { @@ -122,9 +127,9 @@ service CatalogProtocolService { rpc getPartitionByPartitionName(PartitionIdentifierProto) returns (GetPartitionDescResponse); rpc getPartitionsByTableName(PartitionIdentifierProto) returns (GetPartitionsResponse); - rpc getAllPartitions(NullProto) returns (GetPartitionsResponse); + rpc getAllPartitions(NullProto) returns (GetTablePartitionsResponse); rpc addPartitions(AddPartitionsProto) returns (ReturnState); - rpc getAllPartitionKeys(NullProto) returns (GetPartitionKeysResponse); + rpc getAllPartitionKeys(NullProto) returns (GetTablePartitionKeysResponse); rpc createIndex(IndexDescProto) returns (ReturnState); rpc dropIndex(IndexNameProto) returns (ReturnState); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index 8cff2cbd8a..84c0bf406e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -158,18 +158,18 @@ void alterTablespace(AlterTablespaceProto alterTablespace) boolean existPartitionMethod(String databaseName, String tableName) throws UndefinedTableException, UndefinedDatabaseException; - GetPartitionDescProto getPartition(String databaseName, String tableName, String partitionName) + PartitionDescProto getPartition(String databaseName, String tableName, String partitionName) throws UndefinedPartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException, UndefinedTableException; - List getPartitions(String databaseName, String tableName); + List getPartitions(String databaseName, String tableName); - List getAllPartitions(); + List getAllPartitions(); void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws UndefinedTableException, DuplicatePartitionException, UndefinedPartitionMethodException, UndefinedDatabaseException; - List getAllPartitionKeys(); + List getAllPartitionKeys(); boolean createIndex(IndexDesc index); diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index caf94d085b..4fdde26bfc 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -145,6 +145,19 @@ message TableOptionProto { required KeyValueProto keyval = 2; } +message TablePartitionProto { + required int32 partition_id = 1; + required int32 tid = 2; + optional string partitionName = 3; + optional string path = 4; +} + +message TablePartitionKeyProto { + required string columnName = 1; + optional string partitionValue = 2; + required int32 partition_id = 3; +} + message GetFunctionsResponse { required ReturnState state = 1; repeated FunctionDescProto functionDesc = 2; @@ -240,25 +253,12 @@ message PartitionDescProto { required string partitionName = 1; repeated PartitionKeyProto partitionKeys = 2; optional string path = 3; + optional int32 id = 4; } message PartitionKeyProto { required string columnName = 1; - optional string partitionValue = 3; -} - -message GetPartitionDescProto { - required string partitionName = 1; - repeated GetPartitionKeyProto partitionKeys = 2; - optional string path = 3; - required int32 partition_id = 4; - required int32 tid = 5; -} - -message GetPartitionKeyProto { - required string columnName = 1; - optional string partitionValue = 3; - required int32 partition_id = 4; + optional string partitionValue = 2; } message PartitionIdentifierProto { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index aa593d691d..76f8837202 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -595,7 +595,7 @@ public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescPro final String databaseName = split[0]; final String tableName = split[1]; String partitionName = null; - GetPartitionDescProto partitionDesc = null; + CatalogProtos.PartitionDescProto partitionDesc = null; switch (alterTableDescProto.getAlterTableType()) { case RENAME_TABLE: @@ -739,14 +739,15 @@ private void addPartition(String databaseName, String tableName, CatalogProtos.P } } - private void dropPartition(String databaseName, String tableName, GetPartitionDescProto partitionDescProto) { + private void dropPartition(String databaseName, String tableName, CatalogProtos.PartitionDescProto + partitionDescProto) { HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; try { client = clientPool.getClient(); List values = Lists.newArrayList(); - for(GetPartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { + for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { values.add(keyProto.getPartitionValue()); } client.getHiveClient().dropPartition(databaseName, tableName, values, true); @@ -844,40 +845,35 @@ public boolean existPartitionMethod(String databaseName, String tableName) throw } @Override - public List getPartitions(String databaseName, + public List getPartitions(String databaseName, String tableName) { throw new UnsupportedOperationException(); } @Override - public GetPartitionDescProto getPartition(String databaseName, String tableName, + public CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, String partitionName) { HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; - GetPartitionDescProto.Builder builder = null; + CatalogProtos.PartitionDescProto.Builder builder = null; try { client = clientPool.getClient(); Partition partition = client.getHiveClient().getPartition(databaseName, tableName, partitionName); - builder = GetPartitionDescProto.newBuilder(); + builder = CatalogProtos.PartitionDescProto.newBuilder(); builder.setPartitionName(partitionName); builder.setPath(partition.getSd().getLocation()); - // These are dummy values because there is no way to get partition id and table id in HiveMetaStore. - builder.setPartitionId(0); - builder.setTid(0); String[] partitionNames = partitionName.split("/"); for (int i = 0; i < partition.getValues().size(); i++) { String value = partition.getValues().get(i); - GetPartitionKeyProto.Builder keyBuilder = GetPartitionKeyProto.newBuilder(); + CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder(); String columnName = partitionNames[i].split("=")[0]; keyBuilder.setColumnName(columnName); keyBuilder.setPartitionValue(value); - // This is dummy value because there is no way to get partition id in HiveMetaStore. - keyBuilder.setPartitionId(0); builder.addPartitionKeys(keyBuilder); } } catch (NoSuchObjectException e) { @@ -1016,12 +1012,12 @@ public List getAllIndexes() { } @Override - public List getAllPartitions() { + public List getAllPartitions() { throw new UnsupportedOperationException(); } @Override - public List getAllPartitionKeys() { + public List getAllPartitionKeys() { throw new UnsupportedOperationException(); } @@ -1030,7 +1026,7 @@ public void addPartitions(String databaseName, String tableName, List addPartitions = TUtil.newList(); - GetPartitionDescProto existingPartition = null; + CatalogProtos.PartitionDescProto existingPartition = null; try { client = clientPool.getClient(); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index e1dcf9804b..b3af179421 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -29,8 +29,6 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; -import org.apache.tajo.catalog.proto.CatalogProtos.GetPartitionDescProto; -import org.apache.tajo.catalog.proto.CatalogProtos.GetPartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageConstants; @@ -268,7 +266,7 @@ public void testAddTableByPartition() throws Exception { testDropPartition(NATION, "n_nationkey=10/n_date=20150101"); testDropPartition(NATION, "n_nationkey=20/n_date=20150102"); - GetPartitionDescProto partition = store.getPartition(DB_NAME, NATION, "n_nationkey=10/n_date=20150101"); + CatalogProtos.PartitionDescProto partition = store.getPartition(DB_NAME, NATION, "n_nationkey=10/n_date=20150101"); assertNull(partition); partition = store.getPartition(DB_NAME, NATION, "n_nationkey=20/n_date=20150102"); @@ -305,14 +303,14 @@ private void testAddPartition(URI uri, String tableName, String partitionName) t store.alterTable(alterTableDesc.getProto()); - GetPartitionDescProto resultDesc = store.getPartition(DB_NAME, NATION, partitionName); + CatalogProtos.PartitionDescProto resultDesc = store.getPartition(DB_NAME, NATION, partitionName); assertNotNull(resultDesc); assertEquals(resultDesc.getPartitionName(), partitionName); assertEquals(resultDesc.getPath(), uri.toString() + "/" + partitionName); assertEquals(resultDesc.getPartitionKeysCount(), 2); for (int i = 0; i < resultDesc.getPartitionKeysCount(); i++) { - GetPartitionKeyProto keyProto = resultDesc.getPartitionKeys(i); + CatalogProtos.PartitionKeyProto keyProto = resultDesc.getPartitionKeys(i); String[] eachName = partitionNames[i].split("="); assertEquals(keyProto.getPartitionValue(), eachName[1]); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 41ff7c72bf..2c028c4a95 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -932,7 +932,7 @@ public GetPartitionDescResponse getPartitionByPartitionName(RpcController contro rlock.lock(); try { - GetPartitionDescProto partitionDesc = store.getPartition(dbName, tbName, partitionName); + PartitionDescProto partitionDesc = store.getPartition(dbName, tbName, partitionName); if (partitionDesc != null) { return GetPartitionDescResponse.newBuilder() .setState(OK) @@ -980,10 +980,10 @@ public GetPartitionsResponse getPartitionsByTableName(RpcController controller, rlock.lock(); try { - List partitions = store.getPartitions(dbName, tbName); + List partitions = store.getPartitions(dbName, tbName); GetPartitionsResponse.Builder builder = GetPartitionsResponse.newBuilder(); - for (GetPartitionDescProto partition : partitions) { + for (PartitionDescProto partition : partitions) { builder.addPartition(partition); } @@ -1003,19 +1003,21 @@ public GetPartitionsResponse getPartitionsByTableName(RpcController controller, } @Override - public GetPartitionsResponse getAllPartitions(RpcController controller, NullProto request) { + public GetTablePartitionsResponse getAllPartitions(RpcController controller, NullProto request) + throws ServiceException { + rlock.lock(); try { - return GetPartitionsResponse.newBuilder() + return GetTablePartitionsResponse.newBuilder() .setState(OK) - .addAllPartition(store.getAllPartitions()) + .addAllPart(store.getAllPartitions()) .build(); } catch (Throwable t) { printStackTraceIfError(LOG, t); - return GetPartitionsResponse.newBuilder() + return GetTablePartitionsResponse.newBuilder() .setState(returnError(t)) .build(); @@ -1046,11 +1048,11 @@ public ReturnState addPartitions(RpcController controller, AddPartitionsProto re } @Override - public GetPartitionKeysResponse getAllPartitionKeys(RpcController controller, NullProto request) { + public GetTablePartitionKeysResponse getAllPartitionKeys(RpcController controller, NullProto request) { rlock.lock(); try { - return GetPartitionKeysResponse.newBuilder() + return GetTablePartitionKeysResponse.newBuilder() .setState(OK) .addAllPartitionKey(store.getAllPartitionKeys()) .build(); @@ -1058,7 +1060,7 @@ public GetPartitionKeysResponse getAllPartitionKeys(RpcController controller, Nu } catch (Throwable t) { printStackTraceIfError(LOG, t); - return GetPartitionKeysResponse.newBuilder() + return GetTablePartitionKeysResponse.newBuilder() .setState(returnError(t)) .build(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index a7607f2d31..bce9baa6d8 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1012,7 +1012,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) final String databaseName = splitted[0]; final String tableName = splitted[1]; String partitionName = null; - GetPartitionDescProto partitionDesc = null; + CatalogProtos.PartitionDescProto partitionDesc = null; int databaseId = getDatabaseId(databaseName); int tableId = getTableId(databaseId, databaseName, tableName); @@ -1056,7 +1056,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) if (partitionDesc == null) { throw new UndefinedPartitionException(partitionName); } - dropPartition(partitionDesc.getPartitionId()); + dropPartition(partitionDesc.getId()); break; case SET_PROPERTY: setProperties(tableId, alterTableDescProto.getParams()); @@ -2058,7 +2058,7 @@ private void ensurePartitionTable(String tbName, int tableId) } @Override - public GetPartitionDescProto getPartition(String databaseName, String tableName, + public CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, String partitionName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException, UndefinedPartitionException { @@ -2070,7 +2070,7 @@ public GetPartitionDescProto getPartition(String databaseName, String tableName, Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; - GetPartitionDescProto.Builder builder = null; + PartitionDescProto.Builder builder = null; try { String sql = "SELECT PATH, " + COL_PARTITIONS_PK + " FROM " + TB_PARTTIONS + @@ -2087,9 +2087,8 @@ public GetPartitionDescProto getPartition(String databaseName, String tableName, res = pstmt.executeQuery(); if (res.next()) { - builder = GetPartitionDescProto.newBuilder(); - builder.setTid(tableId); - builder.setPartitionId(res.getInt(COL_PARTITIONS_PK)); + builder = PartitionDescProto.newBuilder(); + builder.setId(res.getInt(COL_PARTITIONS_PK)); builder.setPath(res.getString("PATH")); builder.setPartitionName(partitionName); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); @@ -2104,7 +2103,7 @@ public GetPartitionDescProto getPartition(String databaseName, String tableName, return builder.build(); } - private void setPartitionKeys(int pid, GetPartitionDescProto.Builder partitionDesc) { + private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) { Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; @@ -2119,10 +2118,9 @@ private void setPartitionKeys(int pid, GetPartitionDescProto.Builder partitionDe res = pstmt.executeQuery(); while (res.next()) { - GetPartitionKeyProto.Builder builder = GetPartitionKeyProto.newBuilder(); + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); - builder.setPartitionId(pid); partitionDesc.addPartitionKeys(builder); } } catch (SQLException se) { @@ -2133,14 +2131,14 @@ private void setPartitionKeys(int pid, GetPartitionDescProto.Builder partitionDe } @Override - public List getPartitions(String databaseName, String tableName) + public List getPartitions(String databaseName, String tableName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException { Connection conn = null; ResultSet res = null; PreparedStatement pstmt = null; - GetPartitionDescProto.Builder builder = null; - List partitions = new ArrayList(); + PartitionDescProto.Builder builder = null; + List partitions = new ArrayList(); final int databaseId = getDatabaseId(databaseName); final int tableId = getTableId(databaseId, databaseName, tableName); @@ -2160,9 +2158,7 @@ public List getPartitions(String databaseName, String tab res = pstmt.executeQuery(); while (res.next()) { - builder = GetPartitionDescProto.newBuilder(); - builder.setTid(tableId); - builder.setPartitionId(res.getInt(COL_PARTITIONS_PK)); + builder = PartitionDescProto.newBuilder(); builder.setPath(res.getString("PATH")); builder.setPartitionName(res.getString("PARTITION_NAME")); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); @@ -2177,22 +2173,22 @@ public List getPartitions(String databaseName, String tab } @Override - public List getAllPartitions() { + public List getAllPartitions() { Connection conn = null; Statement stmt = null; ResultSet resultSet = null; - List partitions = new ArrayList(); + List partitions = new ArrayList(); try { - String sql = "SELECT A." + COL_PARTITIONS_PK + ", A." + COL_TABLES_PK + ", A.PARTITION_NAME, A.PATH " - + " FROM " + TB_PARTTIONS + " A "; + String sql = "SELECT " + COL_PARTITIONS_PK + ", " + COL_TABLES_PK + ", PARTITION_NAME, " + + " PATH FROM " + TB_PARTTIONS; conn = getConnection(); stmt = conn.createStatement(); resultSet = stmt.executeQuery(sql); while (resultSet.next()) { - GetPartitionDescProto.Builder builder = GetPartitionDescProto.newBuilder(); + TablePartitionProto.Builder builder = TablePartitionProto.newBuilder(); builder.setPartitionId(resultSet.getInt(COL_PARTITIONS_PK)); builder.setTid(resultSet.getInt(COL_TABLES_PK)); @@ -2230,7 +2226,7 @@ public void addPartitions(String databaseName, String tableName, List getAllPartitionKeys() { + public List getAllPartitionKeys() { Connection conn = null; Statement stmt = null; ResultSet resultSet = null; - List partitions = new ArrayList(); + List partitions = new ArrayList(); try { String sql = " SELECT A." + COL_PARTITIONS_PK + ", A.COLUMN_NAME, A.PARTITION_VALUE " + @@ -2340,7 +2336,7 @@ public List getAllPartitionKeys() { stmt = conn.createStatement(); resultSet = stmt.executeQuery(sql); while (resultSet.next()) { - GetPartitionKeyProto.Builder builder = GetPartitionKeyProto.newBuilder(); + TablePartitionKeyProto.Builder builder = TablePartitionKeyProto.newBuilder(); builder.setPartitionId(resultSet.getInt(COL_PARTITIONS_PK)); builder.setColumnName(resultSet.getString("COLUMN_NAME")); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 1b76278418..51ddfbac6d 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -97,17 +97,17 @@ boolean existPartitionMethod(String databaseName, String tableName) throws Undef * @return * @throws TajoException */ - List getPartitions(String databaseName, String tableName) throws + List getPartitions(String databaseName, String tableName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionMethodException; - GetPartitionDescProto getPartition(String databaseName, String tableName, + CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableName, String partitionName) throws UndefinedDatabaseException, UndefinedTableException, UndefinedPartitionException, UndefinedPartitionMethodException; - List getAllPartitions(); + List getAllPartitions(); - List getAllPartitionKeys(); + List getAllPartitionKeys(); void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws UndefinedDatabaseException, diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index d868113c55..8eb53062fe 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -25,7 +25,10 @@ import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.*; +import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; +import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; +import org.apache.tajo.catalog.store.*; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -47,6 +50,7 @@ import java.util.*; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; +import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation; import static org.junit.Assert.*; @@ -896,7 +900,7 @@ public final void testAddAndDeleteTablePartitionByColumn() throws Exception { testAddPartition(tableName, "id=10/name=aaa"); testAddPartition(tableName, "id=20/name=bbb"); - List partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable"); + List partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, "addedtable"); assertNotNull(partitions); assertEquals(partitions.size(), 2); @@ -940,7 +944,7 @@ private void testAddPartition(String tableName, String partitionName) throws Exc catalog.alterTable(alterTableDesc); - GetPartitionDescProto resultDesc = catalog.getPartition(DEFAULT_DATABASE_NAME, + CatalogProtos.PartitionDescProto resultDesc = catalog.getPartition(DEFAULT_DATABASE_NAME, "addedtable", partitionName); assertNotNull(resultDesc); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 0772f08cbc..8339ea71d2 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -25,7 +25,6 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -89,7 +88,7 @@ public final void testAlterTableAddPartition() throws Exception { executeDDL("alter_table_add_partition1.sql", null); executeDDL("alter_table_add_partition2.sql", null); - List partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + List partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 1); assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index abdf767e8c..52e7b54cfa 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -30,7 +30,7 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.TajoClientUtil; import org.apache.tajo.exception.ReturnStateUtil; -import org.apache.tajo.catalog.proto.CatalogProtos.*; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.DataChannel; @@ -1279,7 +1279,7 @@ private void verifyPartitionDirectoryFromCatalog(String databaseName, String tab ResultSet res = executeString(query.toString()); StringBuilder partitionName = new StringBuilder(); - GetPartitionDescProto partitionDescProto = null; + PartitionDescProto partitionDescProto = null; // Check whether that partition's directory exist or doesn't exist. while(res.next()) { @@ -1335,12 +1335,12 @@ public final void testDuplicatedPartitions() throws Exception { // If duplicated partitions had been removed, partitions just will contain 'KEY=N' partition and 'KEY=R' // partition. In previous Query and Stage, duplicated partitions were not deleted because they had been in List. // If you want to verify duplicated partitions, you need to use List instead of Set with DerbyStore. - List partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, tableName); + List partitions = catalog.getPartitions(DEFAULT_DATABASE_NAME, tableName); assertEquals(2, partitions.size()); - GetPartitionDescProto firstPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=N"); + PartitionDescProto firstPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=N"); assertNotNull(firstPartition); - GetPartitionDescProto secondPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=R"); + PartitionDescProto secondPartition = catalog.getPartition(DEFAULT_DATABASE_NAME, tableName, "key=R"); assertNotNull(secondPartition); } finally { executeString("DROP TABLE lineitem2 PURGE"); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index d54010688e..a0f9adc9e5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -27,9 +27,9 @@ import org.apache.tajo.algebra.AlterTablespaceSetType; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; -import org.apache.tajo.catalog.proto.CatalogProtos.GetPartitionDescProto; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.*; @@ -423,7 +423,7 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer Path partitionPath = null; TableDesc desc = null; Pair, String> pair = null; - GetPartitionDescProto partitionDescProto = null; + CatalogProtos.PartitionDescProto partitionDescProto = null; if (alterTable.getAlterTableOpType() == AlterTableOpType.RENAME_TABLE || alterTable.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION @@ -556,7 +556,7 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer } } - private void deletePartitionPath(GetPartitionDescProto partitionDescProto) throws IOException { + private void deletePartitionPath(CatalogProtos.PartitionDescProto partitionDescProto) throws IOException { Path partitionPath = new Path(partitionDescProto.getPath()); FileSystem fs = partitionPath.getFileSystem(context.getConf()); if (fs.exists(partitionPath)) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 5fc281da8d..054fb06149 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -385,12 +385,12 @@ private List getAllTableStats(Schema outSchema) { } private List getAllPartitions(Schema outSchema) { - List partitionList = masterContext.getCatalog().getAllPartitions(); + List partitionList = masterContext.getCatalog().getAllPartitions(); List tuples = new ArrayList(partitionList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; - for (GetPartitionDescProto partition: partitionList) { + for (TablePartitionProto partition: partitionList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { @@ -418,12 +418,12 @@ private List getAllPartitions(Schema outSchema) { } private List getAllPartitionKeys(Schema outSchema) { - List partitionKeyList = masterContext.getCatalog().getAllPartitionKeys(); + List partitionKeyList = masterContext.getCatalog().getAllPartitionKeys(); List tuples = new ArrayList(partitionKeyList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; - for (GetPartitionKeyProto partitionKey: partitionKeyList) { + for (TablePartitionKeyProto partitionKey: partitionKeyList) { aTuple = new VTuple(outSchema.size()); for (int fieldId = 0; fieldId < columns.size(); fieldId++) { From e4702858a406d07ee524f734f93ab6253c8a2b3c Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 4 Sep 2015 15:41:26 +0900 Subject: [PATCH 25/25] Fix unit test bug --- .../tajo/master/TestNonForwardQueryResultSystemScanner.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java index e998fcbf3e..67444ae49d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestNonForwardQueryResultSystemScanner.java @@ -77,7 +77,8 @@ public final void testGetInformationSchema() throws Exception { // Setting select statement for getting partition keys. String selectPartitionKeys = "SELECT count(*) FROM INFORMATION_SCHEMA.PARTITION_KEYS WHERE partition_id IN ("; - resultSet = executeString("SELECT partition_id FROM INFORMATION_SCHEMA.PARTITIONS "); + + resultSet = executeString("SELECT partition_id FROM INFORMATION_SCHEMA.PARTITIONS WHERE tid = " + tid); int i = 0; while (resultSet.next()) { if (i > 0) {