From 241e0668d4c2cefc57502fe0525448c00da5b442 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 14 Jul 2015 23:58:09 +0900 Subject: [PATCH 01/40] TAJO-1346: Run ADD PARTITION after storing column partitioned table. --- .../org/apache/tajo/catalog/CatalogUtil.java | 11 ++ .../catalog/statistics/StatisticsUtil.java | 18 ++- .../tajo/catalog/statistics/TableStats.java | 34 ++++- .../src/main/proto/CatalogProtos.proto | 1 + .../tajo/catalog/store/HiveCatalogStore.java | 5 +- .../tajo/catalog/store/AbstractDBStore.java | 2 +- .../apache/tajo/catalog/store/MemStore.java | 70 +++++----- .../physical/ColPartitionStoreExec.java | 41 ++++++ .../org/apache/tajo/querymaster/Query.java | 14 +- .../engine/query/TestTablePartitions.java | 126 +++++++++++++++++- .../apache/tajo/storage/TableStatistics.java | 6 +- 11 files changed, 284 insertions(+), 44 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 638ebca6a0..7ca40c12c4 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -23,6 +23,8 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.DataTypeUtil; import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.partition.PartitionDesc; +import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; @@ -792,6 +794,15 @@ public static AlterTableDesc setProperty(String tableName, KeyValueSet params, A return alterTableDesc; } + public static AlterTableDesc addPartitionAndDropPartition(String tableName, PartitionDesc partitionDesc, + AlterTableType alterTableType) { + final AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(tableName); + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(alterTableType); + return alterTableDesc; + } + /* It is the relationship graph of type conversions. */ public static final Map> OPERATION_CASTING_MAP = Maps.newHashMap(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java index c481276c33..c86495a877 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.catalog.partition.PartitionDesc; import java.util.List; @@ -49,7 +50,6 @@ public static StatSet aggregateStatSet(List statSets) { * @param stats The TableStats to be aggregated */ public static void aggregateTableStat(TableStats result, TableStats stats) { - if (stats.getColumnStats().size() > 0) { if (result.getColumnStats().size() == 0) { for (int i = 0; i < stats.getColumnStats().size(); i++) { @@ -83,6 +83,14 @@ public static void aggregateTableStat(TableStats result, TableStats stats) { } } + // If there is partitions + if (stats.getPartitions().size() > 0) { + // Aggregate partitions for each table + for (PartitionDesc partitionDesc : stats.getPartitions()) { + result.addPartition(partitionDesc); + } + } + result.setNumRows(result.getNumRows() + stats.getNumRows()); result.setNumBytes(result.getNumBytes() + stats.getNumBytes()); result.setReadBytes(result.getReadBytes() + stats.getReadBytes()); @@ -112,6 +120,14 @@ public static TableStats aggregateTableStat(List tableStatses) { } for (TableStats ts : tableStatses) { + // If there is partitions + if (ts.getPartitions().size() > 0) { + // Aggregate partitions for each table + for (PartitionDesc partitionDesc : ts.getPartitions()) { + aggregated.addPartition(partitionDesc); + } + } + // if there is empty stats if (ts.getColumnStats().size() > 0) { // aggregate column stats for each table diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java index dd358ae1de..8915e8c980 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java @@ -24,6 +24,9 @@ import com.google.common.base.Objects; import com.google.gson.Gson; import com.google.gson.annotations.Expose; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.json.GsonObject; @@ -36,6 +39,8 @@ import java.util.List; public class TableStats implements ProtoObject, Cloneable, GsonObject { + private static Log LOG = LogFactory.getLog(TableStats.class); + @Expose private Long numRows = null; // required @Expose private Long numBytes = null; // required @Expose private Integer numBlocks = null; // optional @@ -43,6 +48,7 @@ public class TableStats implements ProtoObject, Cloneable, Gson @Expose private Long avgRows = null; // optional @Expose private Long readBytes = null; //optional @Expose private List columnStatses = null; // repeated + @Expose private List partitions = null; // repeated public TableStats() { reset(); @@ -56,6 +62,7 @@ public void reset() { avgRows = 0l; readBytes = 0l; columnStatses = TUtil.newList(); + partitions = TUtil.newList(); } public TableStats(CatalogProtos.TableStatsProto proto) { @@ -90,6 +97,11 @@ public TableStats(CatalogProtos.TableStatsProto proto) { } columnStatses.add(new ColumnStats(colProto)); } + + this.partitions = TUtil.newList(); + for (CatalogProtos.PartitionDescProto partitionProto : proto.getPartitionsList()) { + partitions.add(new PartitionDesc(partitionProto)); + } } public Long getNumRows() { @@ -148,10 +160,22 @@ public void setColumnStats(List columnStatses) { this.columnStatses = new ArrayList(columnStatses); } + public List getPartitions() { + return partitions; + } + + public void setPartitions(List partitions) { + this.partitions = partitions; + } + public void addColumnStat(ColumnStats columnStats) { this.columnStatses.add(columnStats); } + public void addPartition(PartitionDesc partitionDesc) { + this.partitions.add(partitionDesc); + } + public boolean equals(Object obj) { if (obj instanceof TableStats) { TableStats other = (TableStats) obj; @@ -163,6 +187,7 @@ public boolean equals(Object obj) { eq = eq && TUtil.checkEquals(this.avgRows, other.avgRows); eq = eq && TUtil.checkEquals(this.readBytes, other.readBytes); eq = eq && TUtil.checkEquals(this.columnStatses, other.columnStatses); + eq = eq && TUtil.checkEquals(this.partitions, other.partitions); return eq; } else { return false; @@ -171,7 +196,7 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hashCode(numRows, numBytes, - numBlocks, numShuffleOutputs, columnStatses); + numBlocks, numShuffleOutputs, columnStatses, partitions); } public Object clone() throws CloneNotSupportedException { @@ -185,6 +210,8 @@ public Object clone() throws CloneNotSupportedException { stat.columnStatses = new ArrayList(this.columnStatses); + stat.partitions = new ArrayList(this.partitions); + return stat; } @@ -260,6 +287,11 @@ public TableStatsProto getProto() { builder.addColStat(colStat.getProto()); } } + if (this.partitions != null) { + for (PartitionDesc partitionDesc: partitions) { + builder.addPartitions(partitionDesc.getProto()); + } + } return builder.build(); } } \ No newline at end of file diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index b213916f30..e0b66dbd61 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -233,6 +233,7 @@ message TableStatsProto { optional int64 readBytes = 7; repeated ColumnStatsProto colStat = 8; optional int32 tid = 9; + repeated PartitionDescProto partitions = 10; } message ColumnStatsProto { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 8f23db4c13..08255e573e 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -621,7 +621,7 @@ public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescPro partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); partitionDesc = getPartition(databaseName, tableName, partitionName); if(partitionDesc != null) { - throw new AlreadyExistsPartitionException(databaseName, tableName, partitionName); + dropPartition(databaseName, tableName, partitionDesc); } addPartition(databaseName, tableName, alterTableDescProto.getPartitionDesc()); break; @@ -630,8 +630,9 @@ public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescPro partitionDesc = getPartition(databaseName, tableName, partitionName); if(partitionDesc == null) { throw new NoSuchPartitionException(databaseName, tableName, partitionName); + } else { + dropPartition(databaseName, tableName, partitionDesc); } - dropPartition(databaseName, tableName, partitionDesc); break; case SET_PROPERTY: // TODO - not implemented yet diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 043c8bc03d..489a6ed9e4 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -999,7 +999,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); partitionDesc = getPartition(databaseName, tableName, partitionName); if(partitionDesc != null) { - throw new AlreadyExistsPartitionException(databaseName, tableName, partitionName); + dropPartition(tableId, alterTableDescProto.getPartitionDesc().getPartitionName()); } addPartition(tableId, alterTableDescProto.getPartitionDesc()); break; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 55c43367a4..b92b556e2d 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -315,41 +315,13 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th partitionName = partitionDesc.getPartitionName(); if (partitions.containsKey(tableName) && partitions.get(tableName).containsKey(partitionName)) { - throw new AlreadyExistsPartitionException(databaseName, tableName, partitionName); - } else { - CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); - builder.setPartitionName(partitionName); - builder.setPath(partitionDesc.getPath()); - - if (partitionDesc.getPartitionKeysCount() > 0) { - int i = 0; - for (CatalogProtos.PartitionKeyProto eachKey : partitionDesc.getPartitionKeysList()) { - CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder(); - keyBuilder.setColumnName(eachKey.getColumnName()); - keyBuilder.setPartitionValue(eachKey.getPartitionValue()); - builder.setPartitionKeys(i, keyBuilder.build()); - i++; - } - } - - Map protoMap = null; - if (!partitions.containsKey(tableName)) { - protoMap = Maps.newHashMap(); - } else { - protoMap = partitions.get(tableName); - } - protoMap.put(partitionName, builder.build()); - partitions.put(tableName, protoMap); + dropPartition(databaseName, tableName, partitionName); } + addPartition(partitionDesc, tableName, partitionName); break; case DROP_PARTITION: - partitionDesc = alterTableDescProto.getPartitionDesc(); partitionName = partitionDesc.getPartitionName(); - if(!partitions.containsKey(tableName)) { - throw new NoSuchPartitionException(databaseName, tableName, partitionName); - } else { - partitions.remove(partitionName); - } + dropPartition(databaseName, tableName, partitionName); break; case SET_PROPERTY: KeyValueSet properties = new KeyValueSet(tableDescProto.getMeta().getParams()); @@ -369,6 +341,42 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th } } + private void addPartition(CatalogProtos.PartitionDescProto partitionDesc, String tableName, String partitionName) { + Map protoMap = null; + + CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); + builder.setPartitionName(partitionName); + builder.setPath(partitionDesc.getPath()); + + if (partitionDesc.getPartitionKeysCount() > 0) { + for (CatalogProtos.PartitionKeyProto eachKey : partitionDesc.getPartitionKeysList()) { + CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder(); + keyBuilder.setColumnName(eachKey.getColumnName()); + keyBuilder.setPartitionValue(eachKey.getPartitionValue()); + builder.addPartitionKeys(keyBuilder.build()); + } + } + + if (!partitions.containsKey(tableName)) { + protoMap = Maps.newHashMap(); + } else { + protoMap = partitions.get(tableName); + } + protoMap.put(partitionName, builder.build()); + partitions.put(tableName, protoMap); + } + + private void dropPartition(String databaseName, String tableName, String partitionName) { + Map protoMap = null; + + if(partitions.containsKey(tableName)) { + protoMap = partitions.get(tableName); + protoMap.remove(partitionName); + partitions.put(tableName, protoMap); + } else { + throw new NoSuchPartitionException(databaseName, tableName, partitionName); + } + } private int getIndexOfColumnToBeRenamed(List fieldList, String columnName) { int fieldCount = fieldList.size(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index 76abc6dbf6..43eeb97c69 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -29,6 +29,8 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.partition.PartitionDesc; +import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.plan.logical.CreateTableNode; import org.apache.tajo.plan.logical.InsertNode; @@ -36,9 +38,12 @@ import org.apache.tajo.plan.logical.StoreTableNode; import org.apache.tajo.storage.*; import org.apache.tajo.unit.StorageUnit; +import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; +import java.net.URI; +import java.util.List; public abstract class ColPartitionStoreExec extends UnaryPhysicalExec { private static Log LOG = LogFactory.getLog(ColPartitionStoreExec.class); @@ -156,9 +161,45 @@ protected Appender getNextPartitionAppender(String partition) throws IOException openAppender(0); + addPartition(partition); + return appender; } + /** + * Add partition information to TableStats for storing to CatalogStore. + * + * @param partition partition name + * @throws IOException + */ + private void addPartition(String partition) throws IOException { + PartitionDesc partitionDesc = new PartitionDesc(); + partitionDesc.setPartitionName(partition); + + String[] partitionKeyPairs = partition.split("/"); + List partitionKeyList = TUtil.newList(); + for(String partitionKeyPair: partitionKeyPairs) { + String[] keyValue = partitionKeyPair.split("="); + PartitionKey partitionKey = new PartitionKey(keyValue[0], keyValue[1]); + partitionKeyList.add(partitionKey); + } + partitionDesc.setPartitionKeys(partitionKeyList); + + if (this.plan.getUri() == null) { + // In CTAS, the uri would be null. So, + String[] split = CatalogUtil.splitTableName(plan.getTableName()); + int endIndex = storeTablePath.toString().indexOf(split[1]) + split[1].length(); + String outputPath = storeTablePath.toString().substring(0, endIndex); + partitionDesc.setPath(outputPath + "/" + partition); + } else { + partitionDesc.setPath(this.plan.getUri().toString() + "/" + partition); + } + + if(!appender.getStats().getPartitions().contains(partitionDesc)) { + appender.getStats().addPartition(partitionDesc); + } + } + public void openAppender(int suffixId) throws IOException { Path actualFilePath = lastFileName; if (suffixId > 0) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 9d5838d6b2..c24ca3f025 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -33,10 +33,9 @@ import org.apache.tajo.QueryVars; import org.apache.tajo.SessionVars; import org.apache.tajo.TajoProtos.QueryState; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto; -import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.ExecutionBlock; @@ -633,8 +632,15 @@ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryCo builder.setStats(stats.getProto()); catalog.updateTableStats(builder.build()); + // If there is partitions + if (stats.getPartitions() != null && stats.getPartitions().size() > 0) { + // Store partitions to CatalogStore using alter table statement. + for (PartitionDesc partitionDesc : stats.getPartitions()) { + catalog.alterTable(CatalogUtil.addPartitionAndDropPartition(finalTable.getName(), partitionDesc, + AlterTableType.ADD_PARTITION)); + } + } } - query.setResultDesc(finalTable); } } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index ef57356c80..b19b206c0e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -31,6 +31,7 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.DataChannel; @@ -73,7 +74,7 @@ public static Collection generateParameters() { return Arrays.asList(new Object[][] { //type {NodeType.INSERT}, - {NodeType.CREATE_TABLE}, +// {NodeType.CREATE_TABLE}, }); } @@ -120,6 +121,10 @@ public final void testCreateColumnPartitionedTable() throws Exception { assertEquals(SCATTERED_HASH_SHUFFLE, channel.getShuffleType()); assertEquals(1, channel.getShuffleKeys().length); + TableDesc tableDesc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"}, + tableDesc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); res.close(); } @@ -166,6 +171,10 @@ public final void testCreateColumnPartitionedTableWithJoin() throws Exception { assertEquals(SCATTERED_HASH_SHUFFLE, channel.getShuffleType()); assertEquals(1, channel.getShuffleKeys().length); + TableDesc tableDesc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"}, + tableDesc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); res.close(); } @@ -192,6 +201,10 @@ public final void testCreateColumnPartitionedTableWithSelectedColumns() throws E } res.close(); + TableDesc tableDesc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"}, + tableDesc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -231,6 +244,10 @@ public final void testColumnPartitionedTableByOneColumn() throws Exception { assertEquals(resultRows1.get(res.getDouble(4))[0], res.getInt(1)); assertEquals(resultRows1.get(res.getDouble(4))[1], res.getInt(2)); } + + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, + new String[]{"key"}, desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); res.close(); } @@ -335,6 +352,9 @@ public final void testQueryCasesOnColumnPartitionedTable() throws Exception { assertResultSet(res, "case13.result"); res.close(); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); res.close(); } @@ -408,6 +428,14 @@ public final void testColumnPartitionedTableByThreeColumns() throws Exception { assertEquals(resultRows2.get(res.getDouble(4))[1], res.getInt(3)); } + res = executeString("SELECT col1, col2, col3 FROM " + tableName); + String result = resultSetToString(res); + System.out.println("### RESULT ### \n:" + result); + res.close(); + + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1", "col2", "col3"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); res.close(); } @@ -435,6 +463,10 @@ public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Ex res.close(); TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1", "col2", "col3"}, + desc.getStats().getNumRows()); + Path path = new Path(desc.getUri()); FileSystem fs = FileSystem.get(conf); @@ -476,6 +508,11 @@ public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Ex res.close(); desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + + // TODO: When inserting into already exists partitioned table, table status need to change correctly. +// verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1", "col2", "col3"}, +// desc.getStats().getNumRows()); + path = new Path(desc.getUri()); verifyDirectoriesForThreeColumns(fs, path, 2); @@ -629,6 +666,9 @@ public final void testColumnPartitionedTableByOneColumnsWithCompression() throws } } + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -685,6 +725,9 @@ public final void testColumnPartitionedTableByTwoColumnsWithCompression() throws } } + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1", "col2"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -780,6 +823,9 @@ public final void testColumnPartitionedTableByThreeColumnsWithCompression() thro res.close(); assertEquals(3, i); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1", "col2", "col3"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -848,6 +894,9 @@ public final void testColumnPartitionedTableNoMatchedPartition() throws Exceptio assertFalse(res.next()); res.close(); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1", "col2", "col3"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -871,6 +920,10 @@ public final void testColumnPartitionedTableWithSmallerExpressions1() throws Exc assertResultSet(res, "case14.result"); res.close(); + TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -897,6 +950,10 @@ public final void testColumnPartitionedTableWithSmallerExpressions2() throws Exc assertResultSet(res, "case15.result"); res.close(); + TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"key"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); } } @@ -979,6 +1036,10 @@ public final void testColumnPartitionedTableWithSmallerExpressions5() throws Exc assertResultSet(res); res.close(); + TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col2"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -1006,6 +1067,10 @@ public final void testColumnPartitionedTableWithSmallerExpressions6() throws Exc assertResultSet(res); res.close(); + TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col2"}, + desc.getStats().getNumRows()); + executeString("DROP TABLE " + tableName + " PURGE").close(); } @@ -1082,6 +1147,10 @@ public void testScatteredHashShuffle() throws Exception { } assertEquals(data.size(), numRows); + TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, "test_partition"); + verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, "test_partition", new String[]{"col1"}, + desc.getStats().getNumRows()); + } finally { testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME.varname, TajoConf.ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME.defaultVal); @@ -1095,7 +1164,6 @@ public void testScatteredHashShuffle() throws Exception { @Test public final void TestSpecialCharPartitionKeys1() throws Exception { // See - TAJO-947: ColPartitionStoreExec can cause URISyntaxException due to special characters. - executeDDL("lineitemspecial_ddl.sql", "lineitemspecial.tbl"); if (nodeType == NodeType.INSERT) { @@ -1175,4 +1243,58 @@ public final void testIgnoreFilesInIntermediateDir() throws Exception { } } + /** + * Verify added partitions to a table. This would check each partition's directory using record of table. + * + * + * @param databaseName + * @param tableName + * @param partitionColumns + * @param numRows + * @throws Exception + */ + private void verifyPartitionDirectoryFromCatalog(String databaseName, String tableName, + String[] partitionColumns, Long numRows) throws Exception { + int rowCount = 0; + + // Get all partition column values + StringBuilder query = new StringBuilder(); + query.append("SELECT"); + for (int i = 0; i < partitionColumns.length; i++) { + String partitionColumn = partitionColumns[i]; + if (i > 0) { + query.append(","); + } + query.append(" ").append(partitionColumn); + } + query.append(" FROM ").append(tableName); + ResultSet res = executeString(query.toString()); + + StringBuilder partitionName = new StringBuilder(); + CatalogProtos.PartitionDescProto partitionDescProto = null; + + // Check whether that partition's directory exist or doesn't exist. + while(res.next()) { + partitionName.delete(0, partitionName.length()); + + for (int i = 0; i < partitionColumns.length; i++) { + String partitionColumn = partitionColumns[i]; + if (i > 0) { + partitionName.append("/"); + } + partitionName.append(partitionColumn).append("=").append(res.getString(partitionColumn)); + } + partitionDescProto = catalog.getPartition(databaseName, tableName, partitionName.toString()); + assertNotNull(partitionDescProto); + assertTrue(partitionDescProto.getPath().indexOf(tableName + "/" + partitionName.toString()) > 0); + rowCount++; + } + + res.close(); + + // Check row count. + if (!testingCluster.isHiveCatalogStoreRunning()) { + assertEquals(numRows, new Long(rowCount)); + } + } } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableStatistics.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableStatistics.java index c101b0bfd7..d00b7e5749 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableStatistics.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TableStatistics.java @@ -42,6 +42,8 @@ public class TableStatistics { private boolean [] comparable; + private TableStats stat; + public TableStatistics(Schema schema) { this.schema = schema; minValues = new VTuple(schema.size()); @@ -59,6 +61,8 @@ public TableStatistics(Schema schema) { comparable[i] = true; } } + + stat = new TableStats(); } public Schema getSchema() { @@ -105,8 +109,6 @@ public void analyzeField(int idx, Tuple tuple) { } public TableStats getTableStat() { - TableStats stat = new TableStats(); - for (int i = 0; i < schema.size(); i++) { Column column = schema.getColumn(i); ColumnStats columnStats = new ColumnStats(column); From 9665501b2b6afaa8a17f874387130007d9780cc7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 15 Jul 2015 08:00:06 +0900 Subject: [PATCH 02/40] Remove unnecessary codes --- .../java/org/apache/tajo/engine/query/TestTablePartitions.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index b19b206c0e..2a271b3b77 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -74,7 +74,7 @@ public static Collection generateParameters() { return Arrays.asList(new Object[][] { //type {NodeType.INSERT}, -// {NodeType.CREATE_TABLE}, + {NodeType.CREATE_TABLE}, }); } @@ -430,7 +430,6 @@ public final void testColumnPartitionedTableByThreeColumns() throws Exception { res = executeString("SELECT col1, col2, col3 FROM " + tableName); String result = resultSetToString(res); - System.out.println("### RESULT ### \n:" + result); res.close(); verifyPartitionDirectoryFromCatalog(DEFAULT_DATABASE_NAME, tableName, new String[]{"col1", "col2", "col3"}, From d22539cdb66b90638ec4f6da98f512997937b98d Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 16 Jul 2015 16:26:23 +0900 Subject: [PATCH 03/40] Add a method to add partitions (not yet implemented interface) --- .../src/main/proto/CatalogProtocol.proto | 1 + .../src/main/proto/CatalogProtos.proto | 6 +++ .../tajo/catalog/store/HiveCatalogStore.java | 5 ++ .../apache/tajo/catalog/CatalogServer.java | 49 +++++++++++++++++-- .../tajo/catalog/store/AbstractDBStore.java | 4 ++ .../tajo/catalog/store/CatalogStore.java | 2 + .../apache/tajo/catalog/store/MemStore.java | 5 ++ 7 files changed, 69 insertions(+), 3 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index 5ace32e7a4..f74612ca2d 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -59,6 +59,7 @@ service CatalogProtocolService { rpc getPartitionByPartitionName(PartitionIdentifierProto) returns (PartitionDescProto); rpc getPartitionsByTableName(PartitionIdentifierProto) returns (PartitionsProto); rpc getAllPartitions(NullProto) returns (GetTablePartitionsProto); + rpc addTablePartitions(AddTablePartitionsProto) returns (BoolProto); rpc createIndex(IndexDescProto) returns (BoolProto); rpc dropIndex(IndexNameProto) returns (BoolProto); diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index e0b66dbd61..5ba7790580 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -169,6 +169,12 @@ message GetTablePartitionsProto { repeated TablePartitionProto part = 1; } +message AddTablePartitionsProto { + required TableIdentifierProto tableIdentifier = 1; + repeated PartitionDescProto partitionDesc = 2; +} + + message IndexProto { required int32 dbId = 1; required int32 tId = 2; diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 08255e573e..515fda6631 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -943,6 +943,11 @@ public List getAllPartitions() throws CatalogException { throw new UnsupportedOperationException(); } + @Override + public void addPartitions(List partitions) throws CatalogException { + + } + @Override public List getAllTableOptions() throws CatalogException { throw new UnsupportedOperationException(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index f2e9795537..b5932dc2f8 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -33,6 +33,7 @@ import org.apache.tajo.catalog.CatalogProtocol.CatalogProtocolService; import org.apache.tajo.catalog.dictionary.InfoSchemaMetadataDictionary; import org.apache.tajo.catalog.exception.*; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.catalog.store.CatalogStore; import org.apache.tajo.catalog.store.DerbyStore; @@ -41,6 +42,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.rpc.BlockingRpcServer; +import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.BoolProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringProto; @@ -743,7 +745,7 @@ public PartitionMethodProto getPartitionMethodByTableName(RpcController controll String tableName = request.getTableName(); if (metaDictionary.isSystemDatabase(databaseName)) { - throw new ServiceException(databaseName + " is a system databsae. It does not contain any partitioned tables."); + throw new ServiceException(databaseName + " is a system database. It does not contain any partitioned tables."); } rlock.lock(); @@ -826,7 +828,7 @@ public PartitionDescProto getPartitionByPartitionName(RpcController controller, String partitionName = request.getPartitionName(); if (metaDictionary.isSystemDatabase(databaseName)) { - throw new ServiceException(databaseName + " is a system databsae. It does not contain any partitioned tables."); + throw new ServiceException(databaseName + " is a system database. It does not contain any partitioned tables."); } rlock.lock(); @@ -868,7 +870,7 @@ public PartitionsProto getPartitionsByTableName(RpcController controller, Partit String tableName = request.getTableName(); if (metaDictionary.isSystemDatabase(databaseName)) { - throw new ServiceException(databaseName + " is a system databsae. It does not contain any partitioned tables."); + throw new ServiceException(databaseName + " is a system database. It does not contain any partitioned tables."); } rlock.lock(); @@ -915,6 +917,47 @@ public GetTablePartitionsProto getAllPartitions(RpcController controller, NullPr } } + @Override + public BoolProto addTablePartitions(RpcController controller, AddTablePartitionsProto request) + throws ServiceException { + + TableIdentifierProto identifier = request.getTableIdentifier(); + String databaseName = identifier.getDatabaseName(); + String tableName = identifier.getTableName(); + + + if (metaDictionary.isSystemDatabase(databaseName)) { + throw new ServiceException(databaseName + " is a system database. It does not contain any partitioned tables."); + } + + rlock.lock(); + try { + boolean contain; + + contain = store.existDatabase(databaseName); + if (contain) { + contain = store.existTable(databaseName, tableName); + if (contain) { + if (store.existPartitionMethod(databaseName, tableName)) { + store.addPartitions(request.getPartitionDescList()); + return ProtoUtil.TRUE; + } else { + throw new NoPartitionedTableException(databaseName, tableName); + } + } else { + throw new NoSuchTableException(tableName); + } + } else { + throw new NoSuchDatabaseException(databaseName); + } + } catch (Exception e) { + LOG.error(e); + throw new ServiceException(e); + } finally { + rlock.unlock(); + } + } + @Override public BoolProto createIndex(RpcController controller, IndexDescProto indexDesc) throws ServiceException { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 489a6ed9e4..5514f0594f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2175,6 +2175,10 @@ public List getAllPartitions() throws CatalogException { return partitions; } + @Override + public void addPartitions(List partitions) throws CatalogException { + + } @Override public void createIndex(final IndexDescProto proto) throws CatalogException { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 57ee74f88a..08b59d6484 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -115,6 +115,8 @@ CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableN List getAllPartitions() throws CatalogException; + void addPartitions(List partitions) throws CatalogException; + /**************************** INDEX *******************************/ void createIndex(IndexDescProto proto) throws CatalogException; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index b92b556e2d..605540c3ff 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -613,6 +613,11 @@ public List getAllPartitions() throws CatalogException { return protos; } + @Override + public void addPartitions(List partitions) throws CatalogException { + + } + /* (non-Javadoc) * @see CatalogStore#createIndex(nta.catalog.proto.CatalogProtos.IndexDescProto) */ From 34e92e1a6ed2ca686e604ce747a8da6f98acc8fb Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 17 Jul 2015 12:07:02 +0900 Subject: [PATCH 04/40] Apply addPartitions to QueryMaster --- .../tajo/catalog/AbstractCatalogClient.java | 22 ++ .../src/main/proto/CatalogProtocol.proto | 2 +- .../apache/tajo/catalog/CatalogService.java | 2 + .../catalog/statistics/StatisticsUtil.java | 10 +- .../tajo/catalog/statistics/TableStats.java | 20 +- .../src/main/proto/CatalogProtos.proto | 2 +- .../tajo/catalog/store/HiveCatalogStore.java | 38 ++- .../apache/tajo/catalog/CatalogServer.java | 6 +- .../tajo/catalog/store/AbstractDBStore.java | 88 +++++- .../tajo/catalog/store/CatalogStore.java | 3 +- .../apache/tajo/catalog/store/MemStore.java | 11 +- .../physical/ColPartitionStoreExec.java | 26 +- .../org/apache/tajo/querymaster/Query.java | 256 +++++++++--------- .../apache/tajo/querymaster/QueryMaster.java | 31 ++- 14 files changed, 339 insertions(+), 178 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 766f6c2627..7c3595f32a 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -342,6 +342,28 @@ public List getAllPartitions() { } } + @Override + public boolean addPartitions(String databaseName, String tableName, List partitions) { + try { + CatalogProtocolService.BlockingInterface stub = getStub(); + AddPartitionsProto.Builder builder = AddPartitionsProto.newBuilder(); + + TableIdentifierProto.Builder identifier = TableIdentifierProto.newBuilder(); + identifier.setDatabaseName(databaseName); + identifier.setTableName(tableName); + builder.setTableIdentifier(identifier.build()); + + for (PartitionDescProto partition: partitions) { + builder.addPartitionDesc(partition); + } + + return stub.addPartitions(null, builder.build()).getValue(); + } catch (ServiceException e) { + LOG.error(e.getMessage(), e); + return false; + } + } + @Override public final Collection getAllTableNames(final String databaseName) { try { diff --git a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto index f74612ca2d..bd2c9f0a3a 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto +++ b/tajo-catalog/tajo-catalog-client/src/main/proto/CatalogProtocol.proto @@ -59,7 +59,7 @@ service CatalogProtocolService { rpc getPartitionByPartitionName(PartitionIdentifierProto) returns (PartitionDescProto); rpc getPartitionsByTableName(PartitionIdentifierProto) returns (PartitionsProto); rpc getAllPartitions(NullProto) returns (GetTablePartitionsProto); - rpc addTablePartitions(AddTablePartitionsProto) returns (BoolProto); + rpc addPartitions(AddPartitionsProto) returns (BoolProto); rpc createIndex(IndexDescProto) returns (BoolProto); rpc dropIndex(IndexNameProto) returns (BoolProto); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index 86b773b704..ac96576caf 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -191,6 +191,8 @@ public interface CatalogService { List getAllPartitions(); + boolean addPartitions(String databaseName, String tableName, List partitions); + boolean createIndex(IndexDesc index); boolean existIndexByName(String databaseName, String indexName); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java index c86495a877..4a8b14f2a7 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java @@ -21,7 +21,7 @@ import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.partition.PartitionDesc; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import java.util.List; @@ -86,8 +86,8 @@ public static void aggregateTableStat(TableStats result, TableStats stats) { // If there is partitions if (stats.getPartitions().size() > 0) { // Aggregate partitions for each table - for (PartitionDesc partitionDesc : stats.getPartitions()) { - result.addPartition(partitionDesc); + for (PartitionDescProto partition : stats.getPartitions()) { + result.addPartition(partition); } } @@ -123,8 +123,8 @@ public static TableStats aggregateTableStat(List tableStatses) { // If there is partitions if (ts.getPartitions().size() > 0) { // Aggregate partitions for each table - for (PartitionDesc partitionDesc : ts.getPartitions()) { - aggregated.addPartition(partitionDesc); + for (PartitionDescProto partition : ts.getPartitions()) { + aggregated.addPartition(partition); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java index 8915e8c980..55c51c72b6 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java @@ -26,8 +26,8 @@ import com.google.gson.annotations.Expose; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.json.GsonObject; import org.apache.tajo.catalog.json.CatalogGsonHelper; @@ -48,7 +48,7 @@ public class TableStats implements ProtoObject, Cloneable, Gson @Expose private Long avgRows = null; // optional @Expose private Long readBytes = null; //optional @Expose private List columnStatses = null; // repeated - @Expose private List partitions = null; // repeated + @Expose private List partitions = null; // repeated public TableStats() { reset(); @@ -99,8 +99,8 @@ public TableStats(CatalogProtos.TableStatsProto proto) { } this.partitions = TUtil.newList(); - for (CatalogProtos.PartitionDescProto partitionProto : proto.getPartitionsList()) { - partitions.add(new PartitionDesc(partitionProto)); + for (PartitionDescProto partition : proto.getPartitionsList()) { + partitions.add(partition); } } @@ -160,11 +160,11 @@ public void setColumnStats(List columnStatses) { this.columnStatses = new ArrayList(columnStatses); } - public List getPartitions() { + public List getPartitions() { return partitions; } - public void setPartitions(List partitions) { + public void setPartitions(List partitions) { this.partitions = partitions; } @@ -172,7 +172,7 @@ public void addColumnStat(ColumnStats columnStats) { this.columnStatses.add(columnStats); } - public void addPartition(PartitionDesc partitionDesc) { + public void addPartition(PartitionDescProto partitionDesc) { this.partitions.add(partitionDesc); } @@ -210,7 +210,7 @@ public Object clone() throws CloneNotSupportedException { stat.columnStatses = new ArrayList(this.columnStatses); - stat.partitions = new ArrayList(this.partitions); + stat.partitions = new ArrayList(this.partitions); return stat; } @@ -288,8 +288,8 @@ public TableStatsProto getProto() { } } if (this.partitions != null) { - for (PartitionDesc partitionDesc: partitions) { - builder.addPartitions(partitionDesc.getProto()); + for (PartitionDescProto partition: partitions) { + builder.addPartitions(partition); } } return builder.build(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 5ba7790580..fa03e6de7b 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -169,7 +169,7 @@ message GetTablePartitionsProto { repeated TablePartitionProto part = 1; } -message AddTablePartitionsProto { +message AddPartitionsProto { required TableIdentifierProto tableIdentifier = 1; repeated PartitionDescProto partitionDesc = 2; } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 515fda6631..5946cd4c43 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -54,6 +54,7 @@ import org.apache.tajo.exception.InternalException; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.TUtil; import org.apache.thrift.TException; import java.io.IOException; @@ -752,7 +753,7 @@ private void dropPartition(String databaseName, String tableName, CatalogProtos. for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { values.add(keyProto.getPartitionValue()); } - client.getHiveClient().dropPartition(databaseName, tableName, values, true); + client.getHiveClient().dropPartition(databaseName, tableName, values, false); } catch (Exception e) { throw new CatalogException(e); } finally { @@ -944,7 +945,40 @@ public List getAllPartitions() throws CatalogException { } @Override - public void addPartitions(List partitions) throws CatalogException { + public void addPartitions(String databaseName, String tableName, List partitions) + throws CatalogException { + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + List addPartitions = TUtil.newList(); + + try { + client = clientPool.getClient(); + + for (CatalogProtos.PartitionDescProto partitionDescProto : partitions) { + Partition partition = new Partition(); + partition.setDbName(databaseName); + partition.setTableName(tableName); + + List values = Lists.newArrayList(); + for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { + values.add(keyProto.getPartitionValue()); + } + partition.setValues(values); + + Table table = client.getHiveClient().getTable(databaseName, tableName); + StorageDescriptor sd = table.getSd(); + sd.setLocation(partitionDescProto.getPath()); + partition.setSd(sd); + + addPartitions.add(partition); + } + client.getHiveClient().add_partitions(addPartitions); + } catch (Exception e) { + throw new CatalogException(e); + } finally { + if (client != null) { + client.release(); + } + } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index b5932dc2f8..674dd57b05 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -33,7 +33,6 @@ import org.apache.tajo.catalog.CatalogProtocol.CatalogProtocolService; import org.apache.tajo.catalog.dictionary.InfoSchemaMetadataDictionary; import org.apache.tajo.catalog.exception.*; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.catalog.store.CatalogStore; import org.apache.tajo.catalog.store.DerbyStore; @@ -42,7 +41,6 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.rpc.BlockingRpcServer; -import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.BoolProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringProto; @@ -918,7 +916,7 @@ public GetTablePartitionsProto getAllPartitions(RpcController controller, NullPr } @Override - public BoolProto addTablePartitions(RpcController controller, AddTablePartitionsProto request) + public BoolProto addPartitions(RpcController controller, AddPartitionsProto request) throws ServiceException { TableIdentifierProto identifier = request.getTableIdentifier(); @@ -939,7 +937,7 @@ public BoolProto addTablePartitions(RpcController controller, AddTablePartitions contain = store.existTable(databaseName, tableName); if (contain) { if (store.existPartitionMethod(databaseName, tableName)) { - store.addPartitions(request.getPartitionDescList()); + store.addPartitions(databaseName, tableName, request.getPartitionDescList()); return ProtoUtil.TRUE; } else { throw new NoPartitionedTableException(databaseName, tableName); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 5514f0594f..1a5c110404 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1001,7 +1001,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th if(partitionDesc != null) { dropPartition(tableId, alterTableDescProto.getPartitionDesc().getPartitionName()); } - addPartition(tableId, alterTableDescProto.getPartitionDesc()); + addPartition(tableId, alterTableDescProto.getPartitionDesc(), null); break; case DROP_PARTITION: partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); @@ -1243,7 +1243,8 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th } } - public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition) throws CatalogException { + public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition, + Connection connection) throws CatalogException { Connection conn = null; PreparedStatement pstmt = null; final String ADD_PARTITION_SQL = @@ -1255,13 +1256,18 @@ public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition + COL_PARTITION_VALUE + ") VALUES (?,?,?)"; try { - if (LOG.isDebugEnabled()) { LOG.debug(ADD_PARTITION_SQL); } - conn = getConnection(); - pstmt = conn.prepareStatement(ADD_PARTITION_SQL); + if (connection == null) { + conn = getConnection(); + conn.setAutoCommit(false); + pstmt = conn.prepareStatement(ADD_PARTITION_SQL); + } else { + pstmt = connection.prepareStatement(ADD_PARTITION_SQL); + } + pstmt.setInt(1, tableId); pstmt.setString(2, partition.getPartitionName()); pstmt.setString(3, partition.getPath()); @@ -1269,12 +1275,27 @@ public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition pstmt.close(); if (partition.getPartitionKeysCount() > 0) { - pstmt = conn.prepareStatement(ADD_PARTITION_KEYS_SQL); + if (conn != null) { + pstmt = conn.prepareStatement(ADD_PARTITION_KEYS_SQL); + } else { + pstmt = connection.prepareStatement(ADD_PARTITION_KEYS_SQL); + } int partitionId = getPartitionId(tableId, partition.getPartitionName()); addPartitionKeys(pstmt, partitionId, partition); pstmt.executeBatch(); } + + if (conn != null) { + conn.commit(); + } } catch (SQLException se) { + if (conn != null) { + try { + conn.rollback(); + } catch (SQLException e) { + LOG.error(e, e); + } + } throw new CatalogException(se); } finally { CatalogUtil.closeQuietly(pstmt); @@ -2176,8 +2197,61 @@ public List getAllPartitions() throws CatalogException { } @Override - public void addPartitions(List partitions) throws CatalogException { + public void addPartitions(String databaseName, String tableName, List partitions) + throws CatalogException { + Connection conn = null; + PreparedStatement pstmt = null; + + try { + int databaseId = getDatabaseId(databaseName); + int tableId = getTableId(databaseId, databaseName, tableName); + + conn = getConnection(); + conn.setAutoCommit(false); + StringBuilder sb = new StringBuilder(); + sb.append("DELETE FROM ").append(TB_PARTTION_KEYS); + sb.append("WHERE ").append(COL_PARTITIONS_PK).append(" IN ("); + sb.append(" SELECT ").append(COL_PARTITIONS_PK).append(" FROM ").append(TB_PARTTIONS); + sb.append(" WHERE PARTITION_NAME = ? )"); + + pstmt = conn.prepareStatement(sb.toString()); + for(PartitionDescProto partition : partitions) { + pstmt.setString(1, partition.getPartitionName()); + pstmt.addBatch(); + pstmt.clearParameters(); + } + pstmt.executeBatch(); + pstmt.close(); + sb.append("DELETE FROM ").append(TB_PARTTIONS); + sb.append(" WHERE PARTITION_NAME = ? )"); + pstmt = conn.prepareStatement(sb.toString()); + for(PartitionDescProto partition : partitions) { + pstmt.setString(1, partition.getPartitionName()); + pstmt.addBatch(); + pstmt.clearParameters(); + } + pstmt.executeBatch(); + pstmt.close(); + + for(PartitionDescProto partition : partitions) { + addPartition(tableId, partition, conn); + } + if (conn != null) { + conn.commit(); + } + } catch (SQLException se) { + if (conn != null) { + try { + conn.rollback(); + } catch (SQLException e) { + LOG.error(e, e); + } + } + throw new CatalogException(se); + } finally { + CatalogUtil.closeQuietly(pstmt); + } } @Override diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 08b59d6484..8bcb782a09 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -115,7 +115,8 @@ CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableN List getAllPartitions() throws CatalogException; - void addPartitions(List partitions) throws CatalogException; + void addPartitions(String databaseName, String tableName, List partitions) + throws CatalogException; /**************************** INDEX *******************************/ void createIndex(IndexDescProto proto) throws CatalogException; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 605540c3ff..52cbe0b5f8 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -614,8 +614,15 @@ public List getAllPartitions() throws CatalogException { } @Override - public void addPartitions(List partitions) throws CatalogException { - + public void addPartitions(String databaseName, String tableName, List partitions) + throws CatalogException { + for(CatalogProtos.PartitionDescProto partition: partitions) { + String partitionName = partition.getPartitionName(); + if (this.partitions.containsKey(tableName) && this.partitions.get(tableName).containsKey(partitionName)) { + dropPartition(databaseName, tableName, partitionName); + } + addPartition(partition, tableName, partitionName); + } } /* (non-Javadoc) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index 43eeb97c69..5b33ea4fed 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -29,8 +29,8 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.partition.PartitionDesc; -import org.apache.tajo.catalog.partition.PartitionKey; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.plan.logical.CreateTableNode; import org.apache.tajo.plan.logical.InsertNode; @@ -173,30 +173,30 @@ protected Appender getNextPartitionAppender(String partition) throws IOException * @throws IOException */ private void addPartition(String partition) throws IOException { - PartitionDesc partitionDesc = new PartitionDesc(); - partitionDesc.setPartitionName(partition); + PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); + builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); - List partitionKeyList = TUtil.newList(); for(String partitionKeyPair: partitionKeyPairs) { - String[] keyValue = partitionKeyPair.split("="); - PartitionKey partitionKey = new PartitionKey(keyValue[0], keyValue[1]); - partitionKeyList.add(partitionKey); + String[] split = partitionKeyPair.split("="); + PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); + keyBuilder.setColumnName(split[0]); + keyBuilder.setPartitionValue(split[1]); + builder.addPartitionKeys(keyBuilder.build()); } - partitionDesc.setPartitionKeys(partitionKeyList); if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, String[] split = CatalogUtil.splitTableName(plan.getTableName()); int endIndex = storeTablePath.toString().indexOf(split[1]) + split[1].length(); String outputPath = storeTablePath.toString().substring(0, endIndex); - partitionDesc.setPath(outputPath + "/" + partition); + builder.setPath(outputPath + "/" + partition); } else { - partitionDesc.setPath(this.plan.getUri().toString() + "/" + partition); + builder.setPath(this.plan.getUri().toString() + "/" + partition); } - if(!appender.getStats().getPartitions().contains(partitionDesc)) { - appender.getStats().addPartition(partitionDesc); + if(!appender.getStats().getPartitions().contains(builder)) { + appender.getStats().addPartition(builder.build()); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index c24ca3f025..79692fb7fa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -33,9 +33,10 @@ import org.apache.tajo.QueryVars; import org.apache.tajo.SessionVars; import org.apache.tajo.TajoProtos.QueryState; -import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto; +import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.ExecutionBlock; @@ -102,100 +103,100 @@ public class Query implements EventHandler { private static final QueryCompletedTransition QUERY_COMPLETED_TRANSITION = new QueryCompletedTransition(); protected static final StateMachineFactory - stateMachineFactory = - new StateMachineFactory - (QueryState.QUERY_NEW) - - // Transitions from NEW state - .addTransition(QueryState.QUERY_NEW, QueryState.QUERY_RUNNING, - QueryEventType.START, - new StartTransition()) - .addTransition(QueryState.QUERY_NEW, QueryState.QUERY_NEW, - QueryEventType.DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(QueryState.QUERY_NEW, QueryState.QUERY_KILLED, - QueryEventType.KILL, - new KillNewQueryTransition()) - .addTransition(QueryState.QUERY_NEW, QueryState.QUERY_ERROR, - QueryEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - - // Transitions from RUNNING state - .addTransition(QueryState.QUERY_RUNNING, QueryState.QUERY_RUNNING, - QueryEventType.STAGE_COMPLETED, - STAGE_COMPLETED_TRANSITION) - .addTransition(QueryState.QUERY_RUNNING, - EnumSet.of(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_FAILED, QueryState.QUERY_KILLED, - QueryState.QUERY_ERROR), - QueryEventType.QUERY_COMPLETED, - QUERY_COMPLETED_TRANSITION) - .addTransition(QueryState.QUERY_RUNNING, QueryState.QUERY_RUNNING, - QueryEventType.DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(QueryState.QUERY_RUNNING, QueryState.QUERY_KILL_WAIT, - QueryEventType.KILL, - new KillAllStagesTransition()) - .addTransition(QueryState.QUERY_RUNNING, QueryState.QUERY_ERROR, - QueryEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - - // Transitions from QUERY_SUCCEEDED state - .addTransition(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_SUCCEEDED, - QueryEventType.DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - // ignore-able transitions - .addTransition(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_SUCCEEDED, - QueryEventType.STAGE_COMPLETED, - STAGE_COMPLETED_TRANSITION) - .addTransition(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_SUCCEEDED, - QueryEventType.KILL) - .addTransition(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_ERROR, - QueryEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - - // Transitions from KILL_WAIT state - .addTransition(QueryState.QUERY_KILL_WAIT, QueryState.QUERY_KILL_WAIT, - QueryEventType.STAGE_COMPLETED, - STAGE_COMPLETED_TRANSITION) - .addTransition(QueryState.QUERY_KILL_WAIT, - EnumSet.of(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_FAILED, QueryState.QUERY_KILLED, - QueryState.QUERY_ERROR), - QueryEventType.QUERY_COMPLETED, - QUERY_COMPLETED_TRANSITION) - .addTransition(QueryState.QUERY_KILL_WAIT, QueryState.QUERY_KILL_WAIT, - QueryEventType.DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(QueryState.QUERY_KILL_WAIT, QueryState.QUERY_ERROR, - QueryEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - // Ignore-able transitions - .addTransition(QueryState.QUERY_KILL_WAIT, EnumSet.of(QueryState.QUERY_KILLED), - QueryEventType.KILL, - QUERY_COMPLETED_TRANSITION) - - // Transitions from FAILED state - .addTransition(QueryState.QUERY_FAILED, QueryState.QUERY_FAILED, - QueryEventType.DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(QueryState.QUERY_FAILED, QueryState.QUERY_ERROR, - QueryEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - // Ignore-able transitions - .addTransition(QueryState.QUERY_FAILED, QueryState.QUERY_FAILED, - QueryEventType.KILL) - - // Transitions from ERROR state - .addTransition(QueryState.QUERY_ERROR, QueryState.QUERY_ERROR, - QueryEventType.DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(QueryState.QUERY_ERROR, QueryState.QUERY_ERROR, - QueryEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - // Ignore-able transitions - .addTransition(QueryState.QUERY_ERROR, QueryState.QUERY_ERROR, - EnumSet.of(QueryEventType.KILL, QueryEventType.STAGE_COMPLETED)) - - .installTopology(); + stateMachineFactory = + new StateMachineFactory + (QueryState.QUERY_NEW) + + // Transitions from NEW state + .addTransition(QueryState.QUERY_NEW, QueryState.QUERY_RUNNING, + QueryEventType.START, + new StartTransition()) + .addTransition(QueryState.QUERY_NEW, QueryState.QUERY_NEW, + QueryEventType.DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(QueryState.QUERY_NEW, QueryState.QUERY_KILLED, + QueryEventType.KILL, + new KillNewQueryTransition()) + .addTransition(QueryState.QUERY_NEW, QueryState.QUERY_ERROR, + QueryEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + + // Transitions from RUNNING state + .addTransition(QueryState.QUERY_RUNNING, QueryState.QUERY_RUNNING, + QueryEventType.STAGE_COMPLETED, + STAGE_COMPLETED_TRANSITION) + .addTransition(QueryState.QUERY_RUNNING, + EnumSet.of(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_FAILED, QueryState.QUERY_KILLED, + QueryState.QUERY_ERROR), + QueryEventType.QUERY_COMPLETED, + QUERY_COMPLETED_TRANSITION) + .addTransition(QueryState.QUERY_RUNNING, QueryState.QUERY_RUNNING, + QueryEventType.DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(QueryState.QUERY_RUNNING, QueryState.QUERY_KILL_WAIT, + QueryEventType.KILL, + new KillAllStagesTransition()) + .addTransition(QueryState.QUERY_RUNNING, QueryState.QUERY_ERROR, + QueryEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + + // Transitions from QUERY_SUCCEEDED state + .addTransition(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_SUCCEEDED, + QueryEventType.DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + // ignore-able transitions + .addTransition(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_SUCCEEDED, + QueryEventType.STAGE_COMPLETED, + STAGE_COMPLETED_TRANSITION) + .addTransition(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_SUCCEEDED, + QueryEventType.KILL) + .addTransition(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_ERROR, + QueryEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + + // Transitions from KILL_WAIT state + .addTransition(QueryState.QUERY_KILL_WAIT, QueryState.QUERY_KILL_WAIT, + QueryEventType.STAGE_COMPLETED, + STAGE_COMPLETED_TRANSITION) + .addTransition(QueryState.QUERY_KILL_WAIT, + EnumSet.of(QueryState.QUERY_SUCCEEDED, QueryState.QUERY_FAILED, QueryState.QUERY_KILLED, + QueryState.QUERY_ERROR), + QueryEventType.QUERY_COMPLETED, + QUERY_COMPLETED_TRANSITION) + .addTransition(QueryState.QUERY_KILL_WAIT, QueryState.QUERY_KILL_WAIT, + QueryEventType.DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(QueryState.QUERY_KILL_WAIT, QueryState.QUERY_ERROR, + QueryEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + // Ignore-able transitions + .addTransition(QueryState.QUERY_KILL_WAIT, EnumSet.of(QueryState.QUERY_KILLED), + QueryEventType.KILL, + QUERY_COMPLETED_TRANSITION) + + // Transitions from FAILED state + .addTransition(QueryState.QUERY_FAILED, QueryState.QUERY_FAILED, + QueryEventType.DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(QueryState.QUERY_FAILED, QueryState.QUERY_ERROR, + QueryEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + // Ignore-able transitions + .addTransition(QueryState.QUERY_FAILED, QueryState.QUERY_FAILED, + QueryEventType.KILL) + + // Transitions from ERROR state + .addTransition(QueryState.QUERY_ERROR, QueryState.QUERY_ERROR, + QueryEventType.DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(QueryState.QUERY_ERROR, QueryState.QUERY_ERROR, + QueryEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + // Ignore-able transitions + .addTransition(QueryState.QUERY_ERROR, QueryState.QUERY_ERROR, + EnumSet.of(QueryEventType.KILL, QueryEventType.STAGE_COMPLETED)) + + .installTopology(); public Query(final QueryMasterTask.QueryMasterTaskContext context, final QueryId id, final long appSubmitTime, @@ -347,11 +348,11 @@ public MasterPlan getPlan() { public StateMachine getStateMachine() { return stateMachine; } - + public void addStage(Stage stage) { stages.put(stage.getId(), stage); } - + public QueryId getId() { return this.id; } @@ -391,7 +392,7 @@ public ExecutionQueue getExecutionQueue() { } public static class StartTransition - implements SingleArcTransition { + implements SingleArcTransition { @Override public void transition(Query query, QueryEvent queryEvent) { @@ -472,11 +473,11 @@ private QueryState finalizeQuery(Query query, QueryCompletedEvent event) { Tablespace space = TablespaceManager.get(queryContext.get(QueryVars.OUTPUT_TABLE_URI, "")).get(); Path finalOutputDir = space.commitTable( - query.context.getQueryContext(), - lastStage.getId(), - lastStage.getMasterPlan().getLogicalPlan(), - lastStage.getSchema(), - tableDesc); + query.context.getQueryContext(), + lastStage.getId(), + lastStage.getMasterPlan().getLogicalPlan(), + lastStage.getSchema(), + tableDesc); QueryHookExecutor hookExecutor = new QueryHookExecutor(query.context.getQueryMasterContext()); hookExecutor.execute(query.context.getQueryContext(), query, event.getExecutionBlockId(), finalOutputDir); @@ -540,11 +541,11 @@ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryCo TableStats stats = lastStage.getResultStats(); TableDesc resultTableDesc = - new TableDesc( - query.getId().toString(), - lastStage.getSchema(), - meta, - finalOutputDir.toUri()); + new TableDesc( + query.getId().toString(), + lastStage.getSchema(), + meta, + finalOutputDir.toUri()); resultTableDesc.setExternal(true); stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir)); @@ -573,11 +574,11 @@ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryCo TableMeta meta = new TableMeta(createTableNode.getStorageType(), createTableNode.getOptions()); TableDesc tableDescTobeCreated = - new TableDesc( - createTableNode.getTableName(), - createTableNode.getTableSchema(), - meta, - finalOutputDir.toUri()); + new TableDesc( + createTableNode.getTableName(), + createTableNode.getTableSchema(), + meta, + finalOutputDir.toUri()); tableDescTobeCreated.setExternal(createTableNode.isExternal()); if (createTableNode.hasPartition()) { @@ -604,7 +605,7 @@ public boolean isEligible(QueryContext queryContext, Query query, ExecutionBlock @Override public void execute(QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) - throws Exception { + throws Exception { CatalogService catalog = context.getWorkerContext().getCatalog(); Stage lastStage = query.getStage(finalExecBlockId); @@ -632,15 +633,8 @@ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryCo builder.setStats(stats.getProto()); catalog.updateTableStats(builder.build()); - // If there is partitions - if (stats.getPartitions() != null && stats.getPartitions().size() > 0) { - // Store partitions to CatalogStore using alter table statement. - for (PartitionDesc partitionDesc : stats.getPartitions()) { - catalog.alterTable(CatalogUtil.addPartitionAndDropPartition(finalTable.getName(), partitionDesc, - AlterTableType.ADD_PARTITION)); - } - } } + query.setResultDesc(finalTable); } } @@ -700,17 +694,17 @@ public void transition(Query query, QueryEvent event) { query.erroredStagesCount++; } else { LOG.error(String.format("Invalid Stage (%s) State %s at %s", - castEvent.getExecutionBlockId().toString(), castEvent.getState().name(), query.getSynchronizedState().name())); + castEvent.getExecutionBlockId().toString(), castEvent.getState().name(), query.getSynchronizedState().name())); query.eventHandler.handle(new QueryEvent(event.getQueryId(), QueryEventType.INTERNAL_ERROR)); } // if a stage is succeeded and a query is running if (castEvent.getState() == StageState.SUCCEEDED && // latest stage succeeded - query.getSynchronizedState() == QueryState.QUERY_RUNNING && // current state is not in KILL_WAIT, FAILED, or ERROR. - !executeNextBlock(query, castEvent.getExecutionBlockId())) { + query.getSynchronizedState() == QueryState.QUERY_RUNNING && // current state is not in KILL_WAIT, FAILED, or ERROR. + !executeNextBlock(query, castEvent.getExecutionBlockId())) { return; } - // if a query is completed due to finished, kill, failure, or error + // if a query is completed due to finished, kill, failure, or error query.eventHandler.handle(new QueryCompletedEvent(castEvent.getExecutionBlockId(), castEvent.getState())); } catch (Throwable t) { LOG.error(t.getMessage(), t); @@ -769,10 +763,10 @@ public void handle(QueryEvent event) { queryState = getSynchronizedState(); } catch (InvalidStateTransitonException e) { LOG.error("Can't handle this event at current state" - + ", type:" + event - + ", oldState:" + oldState.name() - + ", nextState:" + getSynchronizedState().name() - , e); + + ", type:" + event + + ", oldState:" + oldState.name() + + ", nextState:" + getSynchronizedState().name() + , e); eventHandler.handle(new QueryEvent(this.id, QueryEventType.INTERNAL_ERROR)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index 6c5bd2227d..06f347f693 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -30,6 +30,10 @@ import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.tajo.*; +import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.query.QueryContext; @@ -324,6 +328,31 @@ public void stopQuery(QueryId queryId) { return; } + // In INSERT .. SELECT ... queries or CTAS, dynamic partitions should be added to catalog. + Query query = queryMasterTask.getQuery(); + if (query.getState() == TajoProtos.QueryState.QUERY_SUCCEEDED && query.getResultDesc() != null) { + TableDesc desc = query.getResultDesc(); + TableStats stats = desc.getStats(); + + String databaseName; + String simpleTableName; + if (CatalogUtil.isFQTableName(desc.getName())) { + String[] split = CatalogUtil.splitFQTableName(desc.getName()); + databaseName = split[0]; + simpleTableName = split[1]; + } else { + databaseName = queryContext.getCurrentDatabase(); + simpleTableName = desc.getName(); + } + + // If there is partitions + if (stats.getPartitions() != null && stats.getPartitions().size() > 0) { + CatalogService catalog = catalog = queryMasterContext.getWorkerContext().getCatalog(); + // Store partitions to CatalogStore using alter table statement. + catalog.addPartitions(databaseName, simpleTableName, stats.getPartitions()); + } + } + finishedQueryMasterTasks.put(queryId, queryMasterTask); TajoHeartbeat queryHeartbeat = buildTajoHeartBeat(queryMasterTask); @@ -352,7 +381,7 @@ public void stopQuery(QueryId queryId) { } catch (Exception e) { LOG.error(e.getMessage(), e); } - Query query = queryMasterTask.getQuery(); + query = queryMasterTask.getQuery(); if (query != null) { QueryHistory queryHisory = query.getQueryHistory(); if (queryHisory != null) { From d0adf378343a63c5ab933011218aca2b54127ca6 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 17 Jul 2015 17:16:19 +0900 Subject: [PATCH 05/40] Fix an unit test error --- .../src/main/java/org/apache/tajo/catalog/store/MemStore.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 4cd9af7d0e..02ad6a8893 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -321,6 +321,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th } break; case DROP_PARTITION: + partitionDesc = alterTableDescProto.getPartitionDesc(); partitionName = partitionDesc.getPartitionName(); dropPartition(databaseName, tableName, partitionName); break; @@ -367,7 +368,7 @@ private void addPartition(CatalogProtos.PartitionDescProto partitionDesc, String } private void dropPartition(String databaseName, String tableName, String partitionName) { - if(!partitions.containsKey(tableName)) { + if (partitions.containsKey(tableName) && !partitions.get(tableName).containsKey(partitionName)) { throw new NoSuchPartitionException(databaseName, tableName, partitionName); } else { partitions.get(tableName).remove(partitionName); From a6284fdccf945d20c830a1f847fc86365f0ef696 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 17 Jul 2015 19:32:09 +0900 Subject: [PATCH 06/40] Trigger for travis CI build --- .../src/main/java/org/apache/tajo/querymaster/QueryMaster.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index 06f347f693..810added85 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -336,6 +336,7 @@ public void stopQuery(QueryId queryId) { String databaseName; String simpleTableName; + if (CatalogUtil.isFQTableName(desc.getName())) { String[] split = CatalogUtil.splitFQTableName(desc.getName()); databaseName = split[0]; From dfe98ccbb63ba341222152c5de46e19101df3af0 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 20 Jul 2015 17:35:38 +0900 Subject: [PATCH 07/40] Update codes for addPartitions --- .../AlreadyExistsPartitionException.java | 4 +- .../NoPartitionedTableException.java | 2 +- .../tajo/catalog/store/HiveCatalogStore.java | 80 +++++++++++++++++-- .../tajo/catalog/store/AbstractDBStore.java | 6 +- 4 files changed, 81 insertions(+), 11 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java index a7f0d87934..f3f7610c4b 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java @@ -27,11 +27,11 @@ public AlreadyExistsPartitionException(String message) { } public AlreadyExistsPartitionException(String tableName, String partitionName) { - super(String.format("ERROR: partition \"%s already exist in \"%s\"", partitionName, tableName)); + super(String.format("ERROR: partition \"%s\" already exist in \"%s\"", partitionName, tableName)); } public AlreadyExistsPartitionException(String databaseName, String tableName, String partitionName) { - super(String.format("ERROR: partition \"%s already exist in \"%s.%s\"", partitionName, databaseName, tableName)); + super(String.format("ERROR: partition \"%s\" already exist in \"%s.%s\"", partitionName, databaseName, tableName)); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java index eded78922b..6ef147c904 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java @@ -18,7 +18,7 @@ package org.apache.tajo.catalog.exception; -public class NoPartitionedTableException extends Exception { +public class NoPartitionedTableException extends CatalogException { public NoPartitionedTableException() {} diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 5946cd4c43..a4045cadbe 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -490,7 +490,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); } } else if (tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.CSV) - || tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.TEXTFILE)) { + || tableDesc.getMeta().getStoreType().equalsIgnoreCase(BuiltinStorages.TEXT)) { sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); sd.setInputFormat(org.apache.hadoop.mapred.TextInputFormat.class.getName()); sd.setOutputFormat(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat.class.getName()); @@ -676,7 +676,6 @@ private void renameColumn(String databaseName, String tableName, CatalogProtos.A } } client.getHiveClient().alter_table(databaseName, tableName, table); - } catch (NoSuchObjectException nsoe) { } catch (Exception e) { throw new CatalogException(e); @@ -771,12 +770,82 @@ public void addPartitionMethod(CatalogProtos.PartitionMethodProto partitionMetho @Override public CatalogProtos.PartitionMethodProto getPartitionMethod(String databaseName, String tableName) throws CatalogException { - return null; // TODO - not implemented yet + org.apache.hadoop.hive.ql.metadata.Table table; + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + PartitionMethodDesc partitionMethodDesc = null; + try { + try { + client = clientPool.getClient(); + table = HiveCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); + } catch (NoSuchObjectException nsoe) { + throw new NoSuchTableException(tableName); + } catch (Exception e) { + throw new CatalogException(e); + } + + // set partition keys + List partitionKeys = table.getPartitionKeys(); + + if (partitionKeys != null && partitionKeys.size() > 0) { + org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema(); + StringBuilder sb = new StringBuilder(); + if (partitionKeys.size() > 0) { + for (int i = 0; i < partitionKeys.size(); i++) { + FieldSchema fieldSchema = partitionKeys.get(i); + TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType().toString()); + String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + + CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); + expressionSchema.addColumn(new Column(fieldName, dataType)); + if (i > 0) { + sb.append(","); + } + sb.append(fieldSchema.getName()); + } + partitionMethodDesc = new PartitionMethodDesc( + databaseName, + tableName, + PartitionType.COLUMN, + sb.toString(), + expressionSchema); + } + } else { + throw new NoPartitionedTableException(databaseName, tableName); + } + } finally { + if(client != null) client.release(); + } + + return partitionMethodDesc.getProto(); } @Override public boolean existPartitionMethod(String databaseName, String tableName) throws CatalogException { - return false; // TODO - not implemented yet + boolean exist = false; + org.apache.hadoop.hive.ql.metadata.Table table; + HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; + + try { + try { + client = clientPool.getClient(); + table = HiveCatalogUtil.getTable(client.getHiveClient(), databaseName, tableName); + } catch (NoSuchObjectException nsoe) { + throw new NoSuchTableException(tableName); + } catch (Exception e) { + throw new CatalogException(e); + } + + // set partition keys + List partitionKeys = table.getPartitionKeys(); + + if (partitionKeys != null && partitionKeys.size() > 0) { + exist = true; + } + } finally { + if(client != null) client.release(); + } + + return exist; } @Override @@ -952,7 +1021,6 @@ public void addPartitions(String databaseName, String tableName, List Date: Mon, 20 Jul 2015 21:29:38 +0900 Subject: [PATCH 08/40] Add ifNotExists to addPartition (working) --- .../org/apache/tajo/algebra/AlterTable.java | 17 +++++++++++++++-- .../org/apache/tajo/engine/parser/SQLParser.g4 | 4 ++-- .../apache/tajo/engine/parser/SQLAnalyzer.java | 1 + .../tajo/engine/parser/TestSQLAnalyzer.java | 14 ++++++++++++++ .../default/alter_table_add_partition_5.sql | 1 + 5 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/default/alter_table_add_partition_5.sql diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java index 260025f8ac..00ee14e967 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java @@ -54,6 +54,9 @@ public class AlterTable extends Expr { @Expose @SerializedName("IsPurge") private boolean purge; + @Expose @SerializedName("IfNotExists") + private boolean ifNotExists; + public AlterTable(final String tableName) { super(OpType.AlterTable); this.tableName = tableName; @@ -140,10 +143,18 @@ public void setPurge(boolean purge) { this.purge = purge; } + public boolean isIfNotExists() { + return ifNotExists; + } + + public void setIfNotExists(boolean ifNotExists) { + this.ifNotExists = ifNotExists; + } + @Override public int hashCode() { return Objects.hashCode(tableName, newTableName, columnName, newColumnName, addNewColumn, alterTableOpType, - columns, values, location, params, purge + columns, values, location, params, purge, ifNotExists ); } @@ -161,7 +172,8 @@ boolean equalsTo(Expr expr) { TUtil.checkEquals(values, another.values) && TUtil.checkEquals(location, another.location) && TUtil.checkEquals(params, another.params) && - TUtil.checkEquals(purge, another.purge) + TUtil.checkEquals(purge, another.purge) && + TUtil.checkEquals(ifNotExists, another.ifNotExists) ; } @@ -183,6 +195,7 @@ public Object clone() throws CloneNotSupportedException { alter.params = new HashMap(params); } alter.purge = purge; + alter.ifNotExists = ifNotExists; return alter; } } diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 1137b30448..8f65aaba5a 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -1602,8 +1602,8 @@ alter_table_statement : ALTER TABLE table_name RENAME TO table_name | ALTER TABLE table_name RENAME COLUMN column_name TO column_name | ALTER TABLE table_name ADD COLUMN field_element - | ALTER TABLE table_name (if_not_exists)? ADD PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (LOCATION path=Character_String_Literal)? - | ALTER TABLE table_name (if_exists)? DROP PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (PURGE)? + | ALTER TABLE table_name ADD (if_not_exists)? PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (LOCATION path=Character_String_Literal)? + | ALTER TABLE table_name DROP (if_exists)? PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (PURGE)? | ALTER TABLE table_name SET PROPERTY property_list ; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 9f850f9405..618ea29b4f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1848,6 +1848,7 @@ public Expr visitAlter_table_statement(SQLParser.Alter_table_statementContext ct alterTable.setLocation(path); } alterTable.setPurge(checkIfExist(ctx.PURGE())); + alterTable.setIfNotExists(checkIfExist(ctx.if_not_exists())); } if (checkIfExist(ctx.property_list())) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index 0e37b473dd..08e3b3bda5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -434,6 +434,20 @@ public void testAlterTableAddPartition4() throws IOException { assertEquals("TAJO", value1.getValue()); } + @Test + public void testAlterTableAddPartition5() throws IOException { + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_add_partition_4.sql"); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); + assertEquals(1, alterTable.getColumns().length); + assertEquals(1, alterTable.getValues().length); + assertEquals("col1", alterTable.getColumns()[0].getName()); + LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; + assertEquals("TAJO", value1.getValue()); + } + @Test public void testAlterTableDropPartition1() throws IOException { String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_drop_partition_1.sql"); diff --git a/tajo-core/src/test/resources/queries/default/alter_table_add_partition_5.sql b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_5.sql new file mode 100644 index 0000000000..127d9993c3 --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_5.sql @@ -0,0 +1 @@ +ALTER TABLE table1 ADD IF NOT EXISTS PARTITION (col1 = 'TAJO' ) \ No newline at end of file From 0bb6a8878ef6bcd811df4c51a55a209f74d827ef Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 22 Jul 2015 10:41:08 +0900 Subject: [PATCH 09/40] Add UNDEFINED_PARTITION_KEY to ResultCode --- .../exception/NoSuchPartitionException.java | 37 ------------------- ...va => UndefinedPartitionKeyException.java} | 9 +++-- .../apache/tajo/exception/ErrorMessages.java | 1 + tajo-common/src/main/proto/errors.proto | 1 + .../apache/tajo/master/exec/DDLExecutor.java | 4 +- .../testAlterTableAddDropPartition.result | 2 +- 6 files changed, 10 insertions(+), 44 deletions(-) delete mode 100644 tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java rename tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/{NoSuchPartitionKeyException.java => UndefinedPartitionKeyException.java} (77%) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java deleted file mode 100644 index 70e0d26e69..0000000000 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.catalog.exception; - -public class NoSuchPartitionException extends RuntimeException { - - private static final long serialVersionUID = 277182608283894938L; - - public NoSuchPartitionException(String message) { - super(message); - } - - public NoSuchPartitionException(String tableName, String partitionName) { - super(String.format("ERROR: \"%s\" is not the partition of \"%s\".", partitionName, tableName)); - } - - public NoSuchPartitionException(String databaseName, String tableName, String partitionName) { - super(String.format("ERROR: \"%s\" is not the partition of \"%s.%s\".", partitionName, databaseName, tableName)); - } - -} diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/UndefinedPartitionKeyException.java similarity index 77% rename from tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java rename to tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/UndefinedPartitionKeyException.java index 94574dcde2..5e6d20f6aa 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/UndefinedPartitionKeyException.java @@ -18,12 +18,13 @@ package org.apache.tajo.catalog.exception; -public class NoSuchPartitionKeyException extends RuntimeException { +import org.apache.tajo.error.Errors.ResultCode; + +public class UndefinedPartitionKeyException extends CatalogException { private static final long serialVersionUID = 277182608283894939L; - public NoSuchPartitionKeyException(String tableName, String partitionKey) { - super(String.format("ERROR: \"%s\" column is not the partition key of \"%s\".", - partitionKey, tableName)); + public UndefinedPartitionKeyException(String partitionKey) { + super(ResultCode.UNDEFINED_PARTITION_KEY, partitionKey); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index b051eb57fb..c12f466d74 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -63,6 +63,7 @@ public class ErrorMessages { ADD_MESSAGE(UNDEFINED_COLUMN, "column '%s' does not exist", 1); ADD_MESSAGE(UNDEFINED_FUNCTION, "function does not exist: %s", 1); ADD_MESSAGE(UNDEFINED_PARTITION, "partition '%s' does not exist", 1); + ADD_MESSAGE(UNDEFINED_PARTITION_KEY, "'%s' column is not the partition key", 1); ADD_MESSAGE(UNDEFINED_OPERATOR, "operator does not exist: '%s'", 1); ADD_MESSAGE(DUPLICATE_TABLESPACE, "tablespace '%s' already exists", 1); diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto index 1ad9c856c6..180fd5d79d 100644 --- a/tajo-common/src/main/proto/errors.proto +++ b/tajo-common/src/main/proto/errors.proto @@ -108,6 +108,7 @@ enum ResultCode { UNDEFINED_PARTITION = 519; // ? UNDEFINED_PARTITION_METHOD = 520; // ? UNDEFINED_OPERATOR = 521; // SQLState: 42883 (=UNDEFINED_FUNCTION) + UNDEFINED_PARTITION_KEY = 522; // ? DUPLICATE_TABLESPACE = 531; DUPLICATE_DATABASE = 532; // SQLState: 42P04 diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index f6bb4f7c28..a43b411ad2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -523,7 +523,7 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer partitionDescProto = catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); if (partitionDescProto == null) { - throw new NoSuchPartitionException(tableName, pair.getSecond()); + throw new UndefinedPartitionException(pair.getSecond()); } catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), @@ -556,7 +556,7 @@ private void deletePartitionPath(CatalogProtos.PartitionDescProto partitionDescP private boolean ensureColumnPartitionKeys(String tableName, String[] columnNames) { for(String columnName : columnNames) { if (!ensureColumnPartitionKeys(tableName, columnName)) { - throw new NoSuchPartitionKeyException(tableName, columnName); + throw new UndefinedPartitionKeyException(columnName); } } return true; diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result index d01038a32b..b09f134b31 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result @@ -1,5 +1,5 @@ OK -ERROR: "key2" column is not the partition key of "default.testaltertableaddpartition". +ERROR: 'key2' column is not the partition key OK OK ERROR: partition 'key=0.1' does not exist From 8ac6ffed13aa18418d17afc60294f8aedb2563ef Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 22 Jul 2015 10:50:32 +0900 Subject: [PATCH 10/40] Add ASSUMED_PARTITION_DIRECTORY to ResultCode --- ....java => AssumedPartitionDirectoryException.java} | 12 +++++++----- .../org/apache/tajo/exception/ErrorMessages.java | 4 ++++ tajo-common/src/main/proto/errors.proto | 1 + .../org/apache/tajo/master/exec/DDLExecutor.java | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) rename tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/{AlreadyExistsAssumedPartitionDirectoryException.java => AssumedPartitionDirectoryException.java} (70%) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AssumedPartitionDirectoryException.java similarity index 70% rename from tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java rename to tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AssumedPartitionDirectoryException.java index df13f824a6..c4e71c6c3a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AssumedPartitionDirectoryException.java @@ -18,11 +18,13 @@ package org.apache.tajo.catalog.exception; -public class AlreadyExistsAssumedPartitionDirectoryException extends RuntimeException { - private static final long serialVersionUID = 277182608283894931L; +import org.apache.tajo.error.Errors.ResultCode; - public AlreadyExistsAssumedPartitionDirectoryException(String message) { - super(String.format("ERROR: There is a directory which is assumed to be a partitioned directory : %s", message)); - } +public class AssumedPartitionDirectoryException extends CatalogException { + private static final long serialVersionUID = 277182608283894931L; + + public AssumedPartitionDirectoryException(String columnName) { + super(ResultCode.ASSUMED_PARTITION_DIRECTORY, columnName); + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index c12f466d74..160985ffd1 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -89,6 +89,10 @@ public class ErrorMessages { ADD_MESSAGE(MDC_NO_MATCHED_DATATYPE, "no matched type for %s", 1); ADD_MESSAGE(UNKNOWN_DATAFORMAT, "Unknown data format: '%s'", 1); + + ADD_MESSAGE(ASSUMED_PARTITION_DIRECTORY, "There is a directory which is assumed to be a partitioned directory : " + + "'%s'", 1); + } private static void ADD_MESSAGE(ResultCode code, String msgFormat) { diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto index 180fd5d79d..2603d2c318 100644 --- a/tajo-common/src/main/proto/errors.proto +++ b/tajo-common/src/main/proto/errors.proto @@ -123,6 +123,7 @@ enum ResultCode { AMBIGUOUS_TABLE = 541; // ? AMBIGUOUS_COLUMN = 542; // SQLState: 42702; AMBIGUOUS_FUNCTION = 543; // SQLState: 42725; + ASSUMED_PARTITION_DIRECTORY = 544; // ? CANNOT_CAST = 601; // SQLState: 42846 - Cast from source type to target type is not supported. GROUPING_ERROR = 602; // SQLState: 42803 diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index a43b411ad2..d523eb9cef 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -506,7 +506,7 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond()); if (fs.exists(assumedDirectory) && !assumedDirectory.equals(partitionPath)) { - throw new AlreadyExistsAssumedPartitionDirectoryException(assumedDirectory.toString()); + throw new AssumedPartitionDirectoryException(assumedDirectory.toString()); } catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), From c28704b76e199f6a242de8e82a842deb93909e95 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 22 Jul 2015 11:01:46 +0900 Subject: [PATCH 11/40] Add a unit test case for if_not_exists --- .../apache/tajo/engine/parser/TestSQLAnalyzer.java | 7 ++++++- .../org/apache/tajo/plan/logical/AlterTableNode.java | 12 +++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index 08e3b3bda5..bb38dd48ee 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -379,6 +379,7 @@ public void testAlterTableAddPartition1() throws IOException { assertEquals("1", value1.getValue()); LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; assertEquals("2", value2.getValue()); + assertFalse(alterTable.isIfNotExists()); } @Test @@ -397,6 +398,7 @@ public void testAlterTableAddPartition2() throws IOException { LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; assertEquals("2", value2.getValue()); assertEquals(alterTable.getLocation(), "hdfs://xxx.com/warehouse/table1/col1=1/col2=2"); + assertFalse(alterTable.isIfNotExists()); } @Test @@ -418,6 +420,7 @@ public void testAlterTableAddPartition3() throws IOException { LiteralValue value3 = (LiteralValue)alterTable.getValues()[2]; assertEquals("11", value3.getValue()); assertEquals(alterTable.getLocation(), "hdfs://xxx.com/warehouse/table1/col1=2015/col2=01/col3=11"); + assertFalse(alterTable.isIfNotExists()); } @Test @@ -432,11 +435,12 @@ public void testAlterTableAddPartition4() throws IOException { assertEquals("col1", alterTable.getColumns()[0].getName()); LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; assertEquals("TAJO", value1.getValue()); + assertFalse(alterTable.isIfNotExists()); } @Test public void testAlterTableAddPartition5() throws IOException { - String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_add_partition_4.sql"); + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_add_partition_5.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.AlterTable, expr.getType()); AlterTable alterTable = (AlterTable)expr; @@ -446,6 +450,7 @@ public void testAlterTableAddPartition5() throws IOException { assertEquals("col1", alterTable.getColumns()[0].getName()); LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; assertEquals("TAJO", value1.getValue()); + assertTrue(alterTable.isIfNotExists()); } @Test diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java index ecb173a584..8b2e10bedc 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java @@ -51,6 +51,8 @@ public class AlterTableNode extends LogicalNode { private String location; @Expose private boolean isPurge; + @Expose + private boolean ifNotExists; public AlterTableNode(int pid) { super(pid, NodeType.ALTER_TABLE); @@ -158,6 +160,14 @@ public void setPurge(boolean isPurge) { this.isPurge = isPurge; } + public boolean isIfNotExists() { + return ifNotExists; + } + + public void setIfNotExists(boolean ifNotExists) { + this.ifNotExists = ifNotExists; + } + @Override public PlanString getPlanString() { return new PlanString(this); @@ -166,7 +176,7 @@ public PlanString getPlanString() { @Override public int hashCode() { return Objects.hashCode(tableName, addNewColumn, alterTableOpType, columnName, newColumnName, newTableName, - tableName, properties, partitionColumns, partitionValues, location, isPurge); + tableName, properties, partitionColumns, partitionValues, location, isPurge, ifNotExists); } @Override From 0503b22a8cd78198e6c27785b42532b5aa60cc80 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 22 Jul 2015 11:55:23 +0900 Subject: [PATCH 12/40] Add if_not_exists to planner. --- .../apache/tajo/master/exec/DDLExecutor.java | 46 ++++++++++--------- .../tajo/engine/query/TestAlterTable.java | 11 ++++- .../alter_table_add_partition2.sql | 1 + .../org/apache/tajo/plan/LogicalPlanner.java | 1 + .../plan/serder/LogicalNodeDeserializer.java | 2 + tajo-plan/src/main/proto/Plan.proto | 1 + 6 files changed, 39 insertions(+), 23 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition2.sql diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index d523eb9cef..5a9790face 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -487,35 +487,37 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer } catch (UndefinedPartitionException npe) { duplicatedPartition = false; } - if (duplicatedPartition) { - throw new DuplicatePartitionException(pair.getSecond()); - } - if (alterTable.getLocation() != null) { - partitionPath = new Path(alterTable.getLocation()); - } else { - // If location is not specified, the partition's location will be set using the table location. - partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); - alterTable.setLocation(partitionPath.toString()); - } + if (duplicatedPartition && !alterTable.isIfNotExists()) { + throw new DuplicatePartitionException(pair.getSecond()); + } else if (!duplicatedPartition) { + if (alterTable.getLocation() != null) { + partitionPath = new Path(alterTable.getLocation()); + } else { + // If location is not specified, the partition's location will be set using the table location. + partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); + alterTable.setLocation(partitionPath.toString()); + } - FileSystem fs = partitionPath.getFileSystem(context.getConf()); + FileSystem fs = partitionPath.getFileSystem(context.getConf()); - // If there is a directory which was assumed to be a partitioned directory and users don't input another - // location, this will throw exception. - Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond()); + // If there is a directory which was assumed to be a partitioned directory and users don't input another + // location, this will throw exception. + Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond()); - if (fs.exists(assumedDirectory) && !assumedDirectory.equals(partitionPath)) { - throw new AssumedPartitionDirectoryException(assumedDirectory.toString()); - } + if (fs.exists(assumedDirectory) && !assumedDirectory.equals(partitionPath)) { + throw new AssumedPartitionDirectoryException(assumedDirectory.toString()); + } - catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), - alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION)); + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), + alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION)); - // If the partition's path doesn't exist, this would make the directory by force. - if (!fs.exists(partitionPath)) { - fs.mkdirs(partitionPath); + // If the partition's path doesn't exist, this would make the directory by force. + if (!fs.exists(partitionPath)) { + fs.mkdirs(partitionPath); + } } + break; case DROP_PARTITION: ensureColumnPartitionKeys(qualifiedName, alterTable.getPartitionColumns()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 8cdaf80d8a..45c3828100 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -24,6 +24,7 @@ import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.exception.DuplicatePartitionException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -31,7 +32,6 @@ import java.sql.ResultSet; import java.util.List; -import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.*; @Category(IntegrationTest.class) @@ -88,6 +88,15 @@ public final void testAlterTableAddPartition() throws Exception { executeDDL("alter_table_add_partition1.sql", null); + // checking a duplicated partition + boolean duplicatedPartition = false; + try { + executeDDL("alter_table_add_partition2.sql", null); + } catch (DuplicatePartitionException npe) { + duplicatedPartition = true; + } + assertFalse(duplicatedPartition); + List partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 1); diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition2.sql b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition2.sql new file mode 100644 index 0000000000..315ac47408 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition2.sql @@ -0,0 +1 @@ +ALTER TABLE partitioned_table ADD IF NOT EXISTS PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index ef8663d35b..3dafb3eab2 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -2135,6 +2135,7 @@ public LogicalNode visitAlterTable(PlanContext context, Stack stack, Alter } alterTableNode.setPurge(alterTable.isPurge()); + alterTableNode.setIfNotExists(alterTable.isIfNotExists()); alterTableNode.setAlterTableOpType(alterTable.getAlterTableOpType()); return alterTableNode; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index fe900c05ca..d5a2b6ddaf 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -604,6 +604,7 @@ private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) if (alterPartition.getLocation() != null) { alterTable.setLocation(alterPartition.getLocation()); } + alterTable.setIfNotExists(alterPartition.getIfNotExists()); break; case DROP_PARTITION: alterPartition = alterTableProto.getAlterPartition(); @@ -611,6 +612,7 @@ private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) .getColumnNamesCount()])); alterTable.setPartitionValues(alterPartition.getPartitionValuesList().toArray(new String[alterPartition .getPartitionValuesCount()])); + alterTable.setPurge(alterPartition.getPurge()); break; default: throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + alterTableProto.getSetType().name()); diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 537d180a5e..6c3d8b642f 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -303,6 +303,7 @@ message AlterTableNode { repeated string partitionValues = 21; optional string location = 3; optional bool purge = 4; + optional bool ifNotExists = 5; } required string tableName = 1; From 61b908800587469638fc90656a546b83ce1d13a7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 22 Jul 2015 13:17:37 +0900 Subject: [PATCH 13/40] Add if_not_exists to Catalog --- .../java/org/apache/tajo/catalog/AbstractCatalogClient.java | 5 ++++- .../main/java/org/apache/tajo/catalog/CatalogService.java | 3 ++- .../tajo-catalog-common/src/main/proto/CatalogProtos.proto | 1 + .../java/org/apache/tajo/catalog/store/HiveCatalogStore.java | 4 ++-- .../src/main/java/org/apache/tajo/catalog/CatalogServer.java | 2 +- .../java/org/apache/tajo/catalog/store/AbstractDBStore.java | 4 ++-- .../java/org/apache/tajo/catalog/store/CatalogStore.java | 4 ++-- .../main/java/org/apache/tajo/catalog/store/MemStore.java | 4 ++-- .../main/java/org/apache/tajo/querymaster/QueryMaster.java | 2 +- 9 files changed, 17 insertions(+), 12 deletions(-) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 62bdea5d89..98e0514d19 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -402,7 +402,8 @@ public List getAllPartitions() { } @Override - public boolean addPartitions(String databaseName, String tableName, List partitions) { + public boolean addPartitions(String databaseName, String tableName, List partitions + , boolean ifNotExists) { try { final BlockingInterface stub = getStub(); final AddPartitionsProto.Builder builder = AddPartitionsProto.newBuilder(); @@ -416,6 +417,8 @@ public boolean addPartitions(String databaseName, String tableName, List getAllPartitions(); - boolean addPartitions(String databaseName, String tableName, List partitions); + boolean addPartitions(String databaseName, String tableName, List partitions + , boolean ifNotExists); boolean createIndex(IndexDesc index); diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 4cda0d7476..bc9baec263 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -260,6 +260,7 @@ message PartitionMethodProto { message AddPartitionsProto { required TableIdentifierProto tableIdentifier = 1; repeated PartitionDescProto partitionDesc = 2; + required bool ifNotExists = 3; } message PartitionDescProto { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index e4eeddc455..427ea70060 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -1013,8 +1013,8 @@ public List getAllPartitions() throws CatalogException { } @Override - public void addPartitions(String databaseName, String tableName, List partitions) - throws CatalogException { + public void addPartitions(String databaseName, String tableName, List partitions + , boolean ifNotExists) throws CatalogException { HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; List addPartitions = TUtil.newList(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 84e6c0b1b1..2b4f8975f0 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -1084,7 +1084,7 @@ public ReturnState addPartitions(RpcController controller, AddPartitionsProto re contain = store.existTable(databaseName, tableName); if (contain) { if (store.existPartitionMethod(databaseName, tableName)) { - store.addPartitions(databaseName, tableName, request.getPartitionDescList()); + store.addPartitions(databaseName, tableName, request.getPartitionDescList(), request.getIfNotExists()); return OK; } else { return errUndefinedPartitionMethod(tableName); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 3f7c36ceed..53c63993d6 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2193,8 +2193,8 @@ public List getAllPartitions() throws CatalogException { } @Override - public void addPartitions(String databaseName, String tableName, List partitions) - throws CatalogException { + public void addPartitions(String databaseName, String tableName, List partitions + , boolean ifNotExists) throws CatalogException { Connection conn = null; PreparedStatement pstmt = null; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java index 7a375469ad..a1ad494f38 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/CatalogStore.java @@ -117,8 +117,8 @@ CatalogProtos.PartitionDescProto getPartition(String databaseName, String tableN List getAllPartitions() throws CatalogException; - void addPartitions(String databaseName, String tableName, List partitions) - throws CatalogException; + void addPartitions(String databaseName, String tableName, List partitions + , boolean ifNotExists) throws CatalogException; /**************************** INDEX *******************************/ void createIndex(IndexDescProto proto) throws CatalogException; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index b29c23d698..19a67233d8 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -611,8 +611,8 @@ public List getAllPartitions() throws CatalogException { } @Override - public void addPartitions(String databaseName, String tableName, List partitions) - throws CatalogException { + public void addPartitions(String databaseName, String tableName, List partitions + , boolean ifNotExists) throws CatalogException { for(CatalogProtos.PartitionDescProto partition: partitions) { String partitionName = partition.getPartitionName(); if (this.partitions.containsKey(tableName) && this.partitions.get(tableName).containsKey(partitionName)) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index a115de0764..65ba3d1bf2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -312,7 +312,7 @@ public void stopQuery(QueryId queryId) { if (stats.getPartitions() != null && stats.getPartitions().size() > 0) { CatalogService catalog = catalog = queryMasterContext.getWorkerContext().getCatalog(); // Store partitions to CatalogStore using alter table statement. - catalog.addPartitions(databaseName, simpleTableName, stats.getPartitions()); + catalog.addPartitions(databaseName, simpleTableName, stats.getPartitions(), true); } } From a929ffdce8cfa140f55f2ee15381f3ea7d21d286 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 22 Jul 2015 15:31:27 +0900 Subject: [PATCH 14/40] Add ifNotExists to CatalogStore --- .../org/apache/tajo/catalog/CatalogUtil.java | 10 ---- .../tajo/catalog/store/HiveCatalogStore.java | 39 ++++++++----- .../tajo/catalog/store/AbstractDBStore.java | 57 +++++++++++-------- .../apache/tajo/catalog/store/MemStore.java | 7 ++- 4 files changed, 65 insertions(+), 48 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 260de29498..87bd9728eb 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -827,16 +827,6 @@ public static AlterTableDesc addOrDropPartition(String tableName, String[] colum return alterTableDesc; } - public static AlterTableDesc addPartitionAndDropPartition(String tableName, PartitionDesc partitionDesc, - AlterTableType alterTableType) { - final AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(tableName); - alterTableDesc.setPartitionDesc(partitionDesc); - alterTableDesc.setAlterTableType(alterTableType); - return alterTableDesc; - } - - /** * Get partition key/value list and partition name * diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 427ea70060..e01a51683f 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -1017,28 +1017,39 @@ public void addPartitions(String databaseName, String tableName, List addPartitions = TUtil.newList(); + CatalogProtos.PartitionDescProto existingPartition = null; try { client = clientPool.getClient(); for (CatalogProtos.PartitionDescProto partitionDescProto : partitions) { - Partition partition = new Partition(); - partition.setDbName(databaseName); - partition.setTableName(tableName); + existingPartition = getPartition(databaseName, tableName, partitionDescProto.getPartitionName()); + + // Unfortunately, hive client add_partitions doesn't run as expected. The method never read the ifNotExists + // parameter. So, if Tajo add existing partition to Hive, it will threw AlreadyExistsException. To avoid + // above error, we need filter existing partitions before call add_partitions. + if (existingPartition != null) { + Partition partition = new Partition(); + partition.setDbName(databaseName); + partition.setTableName(tableName); + + List values = Lists.newArrayList(); + for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { + values.add(keyProto.getPartitionValue()); + } + partition.setValues(values); - List values = Lists.newArrayList(); - for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) { - values.add(keyProto.getPartitionValue()); - } - partition.setValues(values); + Table table = client.getHiveClient().getTable(databaseName, tableName); + StorageDescriptor sd = table.getSd(); + sd.setLocation(partitionDescProto.getPath()); + partition.setSd(sd); - Table table = client.getHiveClient().getTable(databaseName, tableName); - StorageDescriptor sd = table.getSd(); - sd.setLocation(partitionDescProto.getPath()); - partition.setSd(sd); + addPartitions.add(partition); + } + } - addPartitions.add(partition); + if (addPartitions.size() > 0) { + client.getHiveClient().add_partitions(addPartitions, true, true); } - client.getHiveClient().add_partitions(addPartitions, false, true); } catch (Exception e) { throw new TajoInternalError(e); } finally { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 53c63993d6..90612e9601 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2204,33 +2204,44 @@ public void addPartitions(String databaseName, String tableName, List Date: Wed, 22 Jul 2015 15:50:45 +0900 Subject: [PATCH 15/40] Fix error for Setting PrepareStatement parameters --- .../java/org/apache/tajo/catalog/store/AbstractDBStore.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 90612e9601..9e0c9da245 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -937,9 +937,9 @@ public void updateTableStats(final CatalogProtos.UpdateTableStatsProto statsProt } pstmt = conn.prepareStatement(statSql); - pstmt.setInt(1, tableId); - pstmt.setLong(2, statsProto.getStats().getNumRows()); - pstmt.setLong(3, statsProto.getStats().getNumBytes()); + pstmt.setLong(1, statsProto.getStats().getNumRows()); + pstmt.setLong(2, statsProto.getStats().getNumBytes()); + pstmt.setInt(3, tableId); pstmt.executeUpdate(); } From 625077e0455de9fee2f97c9866f3b63203b9b0ba Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 22 Jul 2015 16:34:13 +0900 Subject: [PATCH 16/40] Add comments for IF NOT EXISTS to ALTER TABLE docs. --- tajo-docs/src/main/sphinx/sql_language/alter_table.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst index 947255cd2f..9838b2202c 100644 --- a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst +++ b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst @@ -79,7 +79,7 @@ ADD PARTITION ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx.com/warehouse/table1/col1=1/col2=2' -You can use ``ALTER TABLE ADD PARTITION`` to add partitions to a table. The location must be a directory inside of which data files reside. If the location doesn't exist on the file system, Tajo will make the location by force. ``ADD PARTITION`` changes the table metadata, but does not load data. If the data does not exist in the partition's location, queries will not return any results. +You can use ``ALTER TABLE ADD PARTITION`` to add partitions to a table. The location must be a directory inside of which data files reside. If the location doesn't exist on the file system, Tajo will make the location by force. ``ADD PARTITION`` changes the table metadata, but does not load data. If the data does not exist in the partition's location, queries will not return any results. An error is thrown if the partition for the table already exists. You can use ``IF NOT EXISTS`` to skip the error. ======================== DROP PARTITION From cf266c0cf9280425448ac07772d6ac2bca2a4a75 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 22 Jul 2015 17:09:47 +0900 Subject: [PATCH 17/40] Add ifExists to LogicalPlan and DDL Executor. --- .../org/apache/tajo/algebra/AlterTable.java | 16 +++++++-- .../tajo/engine/parser/SQLAnalyzer.java | 1 + .../apache/tajo/master/exec/DDLExecutor.java | 33 +++++++++++-------- .../tajo/engine/parser/TestSQLAnalyzer.java | 19 +++++++++++ .../tajo/engine/query/TestAlterTable.java | 12 ++----- .../alter_table_drop_partition2.sql | 1 + .../default/alter_table_drop_partition_4.sql | 1 + .../main/sphinx/sql_language/alter_table.rst | 5 ++- .../org/apache/tajo/plan/LogicalPlanner.java | 1 + .../tajo/plan/logical/AlterTableNode.java | 12 ++++++- .../plan/serder/LogicalNodeDeserializer.java | 1 + tajo-plan/src/main/proto/Plan.proto | 1 + 12 files changed, 75 insertions(+), 28 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition2.sql create mode 100644 tajo-core/src/test/resources/queries/default/alter_table_drop_partition_4.sql diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java index 00ee14e967..f0bd62ad9c 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java @@ -56,6 +56,8 @@ public class AlterTable extends Expr { @Expose @SerializedName("IfNotExists") private boolean ifNotExists; + @Expose @SerializedName("IfExists") + private boolean ifExists; public AlterTable(final String tableName) { super(OpType.AlterTable); @@ -151,10 +153,18 @@ public void setIfNotExists(boolean ifNotExists) { this.ifNotExists = ifNotExists; } + public boolean isIfExists() { + return ifExists; + } + + public void setIfExists(boolean ifExists) { + this.ifExists = ifExists; + } + @Override public int hashCode() { return Objects.hashCode(tableName, newTableName, columnName, newColumnName, addNewColumn, alterTableOpType, - columns, values, location, params, purge, ifNotExists + columns, values, location, params, purge, ifNotExists, ifExists ); } @@ -173,7 +183,8 @@ boolean equalsTo(Expr expr) { TUtil.checkEquals(location, another.location) && TUtil.checkEquals(params, another.params) && TUtil.checkEquals(purge, another.purge) && - TUtil.checkEquals(ifNotExists, another.ifNotExists) + TUtil.checkEquals(ifNotExists, another.ifNotExists) && + TUtil.checkEquals(ifExists, another.ifExists) ; } @@ -196,6 +207,7 @@ public Object clone() throws CloneNotSupportedException { } alter.purge = purge; alter.ifNotExists = ifNotExists; + alter.ifExists = ifExists; return alter; } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index b6da000ddf..ab052fb597 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1844,6 +1844,7 @@ public Expr visitAlter_table_statement(SQLParser.Alter_table_statementContext ct } alterTable.setPurge(checkIfExist(ctx.PURGE())); alterTable.setIfNotExists(checkIfExist(ctx.if_not_exists())); + alterTable.setIfExists(checkIfExist(ctx.if_exists())); } if (checkIfExist(ctx.property_list())) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 5a9790face..0e9b9688f1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -484,7 +484,7 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer boolean duplicatedPartition = true; try { catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); - } catch (UndefinedPartitionException npe) { + } catch (UndefinedPartitionException e) { duplicatedPartition = false; } @@ -522,25 +522,32 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer case DROP_PARTITION: ensureColumnPartitionKeys(qualifiedName, alterTable.getPartitionColumns()); pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), alterTable.getPartitionValues()); - partitionDescProto = catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); - if (partitionDescProto == null) { - throw new UndefinedPartitionException(pair.getSecond()); + boolean undefinedPartition = false; + try { + partitionDescProto = catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); + } catch (UndefinedPartitionException e) { + undefinedPartition = true; } - catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), - alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.DROP_PARTITION)); + if (undefinedPartition && !alterTable.isIfExists()) { + throw new UndefinedPartitionException(pair.getSecond()); + } else if (!undefinedPartition) { + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), + alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.DROP_PARTITION)); - // When dropping partition on an managed table, the data will be delete from file system. - if (!desc.isExternal()) { - deletePartitionPath(partitionDescProto); - } else { - // When dropping partition on an external table, the data in the table will NOT be deleted from the file - // system. But if PURGE is specified, the partition data will be deleted. - if (alterTable.isPurge()) { + // When dropping partition on an managed table, the data will be delete from file system. + if (!desc.isExternal()) { deletePartitionPath(partitionDescProto); + } else { + // When dropping partition on an external table, the data in the table will NOT be deleted from the file + // system. But if PURGE is specified, the partition data will be deleted. + if (alterTable.isPurge()) { + deletePartitionPath(partitionDescProto); + } } } + break; default: //TODO diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index bb38dd48ee..bb14aec6cf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -469,6 +469,7 @@ public void testAlterTableDropPartition1() throws IOException { LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; assertEquals("2", value2.getValue()); assertFalse(alterTable.isPurge()); + assertFalse(alterTable.isIfExists()); } @Test @@ -490,6 +491,7 @@ public void testAlterTableDropPartition2() throws IOException { LiteralValue value3 = (LiteralValue)alterTable.getValues()[2]; assertEquals("11", value3.getValue()); assertFalse(alterTable.isPurge()); + assertFalse(alterTable.isIfExists()); } @Test @@ -505,6 +507,23 @@ public void testAlterTableDropPartition3() throws IOException { LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; assertEquals("TAJO", value1.getValue()); assertTrue(alterTable.isPurge()); + assertFalse(alterTable.isIfExists()); + } + + @Test + public void testAlterTableDropPartition4() throws IOException { + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_drop_partition_4.sql"); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.DROP_PARTITION); + assertEquals(1, alterTable.getColumns().length); + assertEquals(1, alterTable.getValues().length); + assertEquals("col1", alterTable.getColumns()[0].getName()); + LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; + assertEquals("TAJO", value1.getValue()); + assertTrue(alterTable.isPurge()); + assertTrue(alterTable.isIfExists()); } @Test diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 45c3828100..4053c0df61 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -25,6 +25,7 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.exception.DuplicatePartitionException; +import org.apache.tajo.catalog.exception.UndefinedPartitionException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -87,15 +88,7 @@ public final void testAlterTableAddPartition() throws Exception { assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(1).getSimpleName(), "col4"); executeDDL("alter_table_add_partition1.sql", null); - - // checking a duplicated partition - boolean duplicatedPartition = false; - try { - executeDDL("alter_table_add_partition2.sql", null); - } catch (DuplicatePartitionException npe) { - duplicatedPartition = true; - } - assertFalse(duplicatedPartition); + executeDDL("alter_table_add_partition2.sql", null); List partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); @@ -113,6 +106,7 @@ public final void testAlterTableAddPartition() throws Exception { assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); executeDDL("alter_table_drop_partition1.sql", null); + executeDDL("alter_table_drop_partition2.sql", null); partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition2.sql b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition2.sql new file mode 100644 index 0000000000..0d4c93298b --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition2.sql @@ -0,0 +1 @@ +ALTER TABLE partitioned_table DROP IF EXISTS PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_4.sql b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_4.sql new file mode 100644 index 0000000000..44c7977594 --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_4.sql @@ -0,0 +1 @@ +ALTER TABLE table1 DROP IF EXISTS PARTITION (col1 = 'TAJO' ) PURGE \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst index 9838b2202c..8923022cf0 100644 --- a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst +++ b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst @@ -89,12 +89,11 @@ You can use ``ALTER TABLE ADD PARTITION`` to add partitions to a table. The loca .. code-block:: sql - ALTER TABLE [IF NOT EXISTS] DROP PARTITION ( = , ...) [PURGE] + ALTER TABLE [IF EXISTS] DROP PARTITION ( = , ...) [PURGE] For example: ALTER TABLE table1 DROP PARTITION (col1 = 1 , col2 = 2) ALTER TABLE table1 DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) PURGE -You can use ``ALTER TABLE DROP PARTITION`` to drop a partition for a table. This removes the data for a managed table - and this doesn't remove the data for an external table. But if ``PURGE`` is specified for an external table, the partition data will be removed. The metadata is completely lost in all cases. \ No newline at end of file +You can use ``ALTER TABLE DROP PARTITION`` to drop a partition for a table. This removes the data for a managed table and this doesn't remove the data for an external table. But if ``PURGE`` is specified for an external table, the partition data will be removed. The metadata is completely lost in all cases. An error is thrown if the partition for the table doesn't exists. You can use ``IF EXISTS`` to skip the error. \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 8db108cfd3..1cf6001508 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -2143,6 +2143,7 @@ public LogicalNode visitAlterTable(PlanContext context, Stack stack, Alter alterTableNode.setPurge(alterTable.isPurge()); alterTableNode.setIfNotExists(alterTable.isIfNotExists()); + alterTableNode.setIfExists(alterTable.isIfExists()); alterTableNode.setAlterTableOpType(alterTable.getAlterTableOpType()); return alterTableNode; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java index 8b2e10bedc..4e25baad00 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java @@ -53,6 +53,8 @@ public class AlterTableNode extends LogicalNode { private boolean isPurge; @Expose private boolean ifNotExists; + @Expose + private boolean ifExists; public AlterTableNode(int pid) { super(pid, NodeType.ALTER_TABLE); @@ -168,6 +170,14 @@ public void setIfNotExists(boolean ifNotExists) { this.ifNotExists = ifNotExists; } + public boolean isIfExists() { + return ifExists; + } + + public void setIfExists(boolean ifExists) { + this.ifExists = ifExists; + } + @Override public PlanString getPlanString() { return new PlanString(this); @@ -176,7 +186,7 @@ public PlanString getPlanString() { @Override public int hashCode() { return Objects.hashCode(tableName, addNewColumn, alterTableOpType, columnName, newColumnName, newTableName, - tableName, properties, partitionColumns, partitionValues, location, isPurge, ifNotExists); + tableName, properties, partitionColumns, partitionValues, location, isPurge, ifNotExists, ifExists); } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index d5a2b6ddaf..054b29fd90 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -613,6 +613,7 @@ private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) alterTable.setPartitionValues(alterPartition.getPartitionValuesList().toArray(new String[alterPartition .getPartitionValuesCount()])); alterTable.setPurge(alterPartition.getPurge()); + alterTable.setIfExists(alterPartition.getIfExists()); break; default: throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + alterTableProto.getSetType().name()); diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 6c3d8b642f..16fde14eac 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -304,6 +304,7 @@ message AlterTableNode { optional string location = 3; optional bool purge = 4; optional bool ifNotExists = 5; + optional bool ifExists = 6; } required string tableName = 1; From 33e061c511036ddf6d6b016256d5da6b8761c06d Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 22 Jul 2015 18:16:19 +0900 Subject: [PATCH 18/40] Add log message for adding partitions. --- .../engine/planner/physical/ColPartitionStoreExec.java | 4 +++- .../java/org/apache/tajo/querymaster/QueryMaster.java | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index 5b33ea4fed..27ead024f1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -150,7 +150,9 @@ protected Appender getNextPartitionAppender(String partition) throws IOException LOG.info("Path " + lastFileName.getParent() + " already exists!"); } else { fs.mkdirs(lastFileName.getParent()); - LOG.info("Add subpartition path directory :" + lastFileName.getParent()); + if (LOG.isDebugEnabled()) { + LOG.debug("Add subpartition path directory :" + lastFileName.getParent()); + } } if (fs.exists(lastFileName)) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index 65ba3d1bf2..46ec61cc1c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -312,7 +312,14 @@ public void stopQuery(QueryId queryId) { if (stats.getPartitions() != null && stats.getPartitions().size() > 0) { CatalogService catalog = catalog = queryMasterContext.getWorkerContext().getCatalog(); // Store partitions to CatalogStore using alter table statement. - catalog.addPartitions(databaseName, simpleTableName, stats.getPartitions(), true); + boolean result = catalog.addPartitions(databaseName, simpleTableName, stats.getPartitions(), true); + if (result) { + LOG.info(String.format("Complete adding for partition %s", stats.getPartitions().size())); + } else { + LOG.info(String.format("Incomplete adding for partition %s", stats.getPartitions().size())); + } + } else { + LOG.info("Can't find partitions for adding."); } } From 51f6fd924199ad661eb7a7a203b445f712d335d9 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 23 Jul 2015 11:58:18 +0900 Subject: [PATCH 19/40] Improve addPartitions performance --- .../src/main/proto/CatalogProtos.proto | 1 + .../apache/tajo/catalog/CatalogServer.java | 11 +- .../tajo/catalog/store/AbstractDBStore.java | 212 +++++++----------- .../apache/tajo/querymaster/QueryMaster.java | 2 + 4 files changed, 93 insertions(+), 133 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index bc9baec263..54581ba767 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -267,6 +267,7 @@ message PartitionDescProto { required string partitionName = 1; repeated PartitionKeyProto partitionKeys = 2; optional string path = 3; + optional int32 id = 4; } message PartitionKeyProto { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 2b4f8975f0..d9730d76bc 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -948,13 +948,16 @@ public GetPartitionDescResponse getPartitionByPartitionName(RpcController contro if (store.existPartitionMethod(dbName, tbName)) { PartitionDescProto partitionDesc = store.getPartition(dbName, tbName, partitionName); - - - return GetPartitionDescResponse.newBuilder() + if (partitionDesc != null) { + return GetPartitionDescResponse.newBuilder() .setState(OK) .setPartition(partitionDesc) .build(); - + } else { + return GetPartitionDescResponse.newBuilder() + .setState(errUndefinedPartition(partitionName)) + .build(); + } } else { return GetPartitionDescResponse.newBuilder() .setState(errUndefinedPartitionMethod(tbName)) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 9e0c9da245..cc6d9a181f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1001,7 +1001,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th if(partitionDesc != null) { throw new DuplicatePartitionException(partitionName); } - addPartition(tableId, alterTableDescProto.getPartitionDesc(), null); + addPartition(tableId, alterTableDescProto.getPartitionDesc()); break; case DROP_PARTITION: partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); @@ -1009,7 +1009,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th if(partitionDesc == null) { throw new UndefinedPartitionException(partitionName); } - dropPartition(tableId, alterTableDescProto.getPartitionDesc().getPartitionName()); + dropPartition(partitionDesc.getId()); break; case SET_PROPERTY: setProperties(tableId, alterTableDescProto.getParams()); @@ -1243,47 +1243,17 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th } } - public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition, - Connection connection) throws CatalogException { + public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition) throws CatalogException { Connection conn = null; - PreparedStatement pstmt = null; - final String ADD_PARTITION_SQL = - "INSERT INTO " + TB_PARTTIONS - + " (" + COL_TABLES_PK + ", PARTITION_NAME, PATH) VALUES (?,?,?)"; - - final String ADD_PARTITION_KEYS_SQL = - "INSERT INTO " + TB_PARTTION_KEYS + " (" + COL_PARTITIONS_PK + ", " + COL_COLUMN_NAME + ", " - + COL_PARTITION_VALUE + ") VALUES (?,?,?)"; + Statement stmt = null; try { - if (LOG.isDebugEnabled()) { - LOG.debug(ADD_PARTITION_SQL); - } - - if (connection == null) { - conn = getConnection(); - conn.setAutoCommit(false); - pstmt = conn.prepareStatement(ADD_PARTITION_SQL); - } else { - pstmt = connection.prepareStatement(ADD_PARTITION_SQL); - } - - pstmt.setInt(1, tableId); - pstmt.setString(2, partition.getPartitionName()); - pstmt.setString(3, partition.getPath()); - pstmt.executeUpdate(); - pstmt.close(); + conn = getConnection(); + conn.setAutoCommit(false); + stmt = conn.createStatement(); - if (partition.getPartitionKeysCount() > 0) { - if (conn != null) { - pstmt = conn.prepareStatement(ADD_PARTITION_KEYS_SQL); - } else { - pstmt = connection.prepareStatement(ADD_PARTITION_KEYS_SQL); - } - int partitionId = getPartitionId(tableId, partition.getPartitionName()); - addPartitionKeys(pstmt, partitionId, partition); - pstmt.executeBatch(); - } + addPartition(tableId, partition, stmt, new StringBuilder()); + stmt.executeBatch(); if (conn != null) { conn.commit(); @@ -1298,84 +1268,85 @@ public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition } throw new TajoInternalError(se); } finally { - CatalogUtil.closeQuietly(pstmt); + CatalogUtil.closeQuietly(stmt); } } - public int getPartitionId(int tableId, String partitionName) throws CatalogException { - Connection conn = null; - ResultSet res = null; - PreparedStatement pstmt = null; - int retValue = -1; - + public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition, + Statement stmt, StringBuilder sb) throws CatalogException { try { - String sql = "SELECT " + COL_PARTITIONS_PK + " FROM " + TB_PARTTIONS + - " WHERE " + COL_TABLES_PK + " = ? AND PARTITION_NAME = ? "; + sb.delete(0, sb.length()); - if (LOG.isDebugEnabled()) { - LOG.debug(sql); - } + sb.append("INSERT INTO ").append(TB_PARTTIONS).append(" "); + sb.append(" (").append(COL_TABLES_PK).append(", PARTITION_NAME, PATH)"); + sb.append(" VALUES (").append(tableId).append(", '").append(partition.getPartitionName()); + sb.append("' , '").append(partition.getPath()).append("')"); + stmt.addBatch(sb.toString()); - conn = getConnection(); - pstmt = conn.prepareStatement(sql); - pstmt.setInt(1, tableId); - pstmt.setString(2, partitionName); - res = pstmt.executeQuery(); - - if (res.next()) { - retValue = res.getInt(1); + if (partition.getPartitionKeysCount() > 0) { + for (int i = 0; i < partition.getPartitionKeysCount(); i++) { + PartitionKeyProto partitionKey = partition.getPartitionKeys(i); + + sb.delete(0, sb.length()); + sb.append("INSERT INTO ").append(TB_PARTTION_KEYS).append(" (").append(COL_PARTITIONS_PK); + sb.append(", ").append(COL_COLUMN_NAME).append(", ").append(COL_PARTITION_VALUE).append(")"); + sb.append(" VALUES ( ("); + sb.append("SELECT ").append(COL_PARTITIONS_PK).append(" FROM ").append(TB_PARTTIONS); + sb.append(" WHERE ").append(COL_TABLES_PK).append(" = ").append(tableId); + sb.append(" AND PARTITION_NAME = '").append(partition.getPartitionName()).append("'"); + sb.append(" ), '").append(partitionKey.getColumnName()).append("'"); + sb.append(", '").append(partitionKey.getPartitionValue()).append("' )"); + + stmt.addBatch(sb.toString()); + } } + } catch (SQLException se) { throw new TajoInternalError(se); - } finally { - CatalogUtil.closeQuietly(pstmt, res); } - return retValue; } - private void addPartitionKeys(PreparedStatement pstmt, int partitionId, PartitionDescProto partition) throws - SQLException { - for (int i = 0; i < partition.getPartitionKeysCount(); i++) { - PartitionKeyProto partitionKey = partition.getPartitionKeys(i); + private void dropPartition(int partitionId) throws CatalogException { + Connection conn = null; + Statement stmt = null; - pstmt.setInt(1, partitionId); - pstmt.setString(2, partitionKey.getColumnName()); - pstmt.setString(3, partitionKey.getPartitionValue()); + try { + conn = getConnection(); + stmt = conn.createStatement(); + dropPartition(partitionId, stmt, new StringBuilder()); - pstmt.addBatch(); - pstmt.clearParameters(); + if(stmt != null) { + stmt.executeBatch(); + } + } catch (SQLException se) { + if (conn != null) { + try { + conn.rollback(); + } catch (SQLException e) { + LOG.error(e, e); + } + } + throw new TajoInternalError(se); + } finally { + CatalogUtil.closeQuietly(stmt); } } - private void dropPartition(int tableId, String partitionName) throws CatalogException { - Connection conn = null; - PreparedStatement pstmt = null; - + private void dropPartition(int partitionId, Statement stmt, StringBuilder sb) throws CatalogException { try { - int partitionId = getPartitionId(tableId, partitionName); - - String sqlDeletePartitionKeys = "DELETE FROM " + TB_PARTTION_KEYS + " WHERE " + COL_PARTITIONS_PK + " = ? "; - String sqlDeletePartition = "DELETE FROM " + TB_PARTTIONS + " WHERE " + COL_PARTITIONS_PK + " = ? "; - - if (LOG.isDebugEnabled()) { - LOG.debug(sqlDeletePartitionKeys); - } - - conn = getConnection(); - pstmt = conn.prepareStatement(sqlDeletePartitionKeys); - pstmt.setInt(1, partitionId); - pstmt.executeUpdate(); - pstmt.close(); + sb.delete(0, sb.length()); + sb.append("DELETE FROM ").append(TB_PARTTION_KEYS); + sb.append(" WHERE ").append(COL_PARTITIONS_PK).append(" = ").append(partitionId); + stmt.addBatch(sb.toString()); - pstmt = conn.prepareStatement(sqlDeletePartition); - pstmt.setInt(1, partitionId); - pstmt.executeUpdate(); + sb.delete(0, sb.length()); + sb.append("DELETE FROM ").append(TB_PARTTIONS); + sb.append(" WHERE ").append(COL_PARTITIONS_PK).append(" = ").append(partitionId); + stmt.addBatch(sb.toString()); } catch (SQLException se) { throw new TajoInternalError(se); - } finally { - CatalogUtil.closeQuietly(pstmt); } } @@ -2077,6 +2048,7 @@ public CatalogProtos.PartitionDescProto getPartition(String databaseName, String if (res.next()) { builder = PartitionDescProto.newBuilder(); + builder.setId(res.getInt(COL_PARTITIONS_PK)); builder.setPath(res.getString("PATH")); builder.setPartitionName(partitionName); setPartitionKeys(res.getInt(COL_PARTITIONS_PK), builder); @@ -2196,7 +2168,9 @@ public List getAllPartitions() throws CatalogException { public void addPartitions(String databaseName, String tableName, List partitions , boolean ifNotExists) throws CatalogException { Connection conn = null; - PreparedStatement pstmt = null; + Statement stmt = null; + PartitionDescProto partitionDesc = null; + StringBuilder sb = null; try { int databaseId = getDatabaseId(databaseName); @@ -2205,47 +2179,27 @@ public void addPartitions(String databaseName, String tableName, List 0) { + LOG.info("### Adding partitions START ###"); CatalogService catalog = catalog = queryMasterContext.getWorkerContext().getCatalog(); // Store partitions to CatalogStore using alter table statement. boolean result = catalog.addPartitions(databaseName, simpleTableName, stats.getPartitions(), true); @@ -318,6 +319,7 @@ public void stopQuery(QueryId queryId) { } else { LOG.info(String.format("Incomplete adding for partition %s", stats.getPartitions().size())); } + LOG.info("### Adding partitions END ###"); } else { LOG.info("Can't find partitions for adding."); } From eca30fe1eb9a604b187cc3bb5ce9a731702bac0b Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 23 Jul 2015 18:05:32 +0900 Subject: [PATCH 20/40] Add batchSize for adding partitions. --- .../tajo/catalog/store/AbstractDBStore.java | 54 +++++++++++++++++-- .../apache/tajo/querymaster/QueryMaster.java | 16 +++++- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index cc6d9a181f..705f173c22 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1253,6 +1253,7 @@ public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition stmt = conn.createStatement(); addPartition(tableId, partition, stmt, new StringBuilder()); + addPartitionKeys(tableId, partition, stmt, new StringBuilder()); stmt.executeBatch(); if (conn != null) { @@ -1281,7 +1282,16 @@ public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition sb.append(" (").append(COL_TABLES_PK).append(", PARTITION_NAME, PATH)"); sb.append(" VALUES (").append(tableId).append(", '").append(partition.getPartitionName()); sb.append("' , '").append(partition.getPath()).append("')"); + stmt.addBatch(sb.toString()); + } catch (SQLException se) { + throw new TajoInternalError(se); + } + } + + public void addPartitionKeys(int tableId, CatalogProtos.PartitionDescProto partition, + Statement stmt, StringBuilder sb) throws CatalogException { + try { if (partition.getPartitionKeysCount() > 0) { for (int i = 0; i < partition.getPartitionKeysCount(); i++) { @@ -2182,22 +2192,58 @@ public void addPartitions(String databaseName, String tableName, List= lastIndex + batchSize && lastIndex != i) { + int[] result = stmt.executeBatch(); + stmt.clearBatch(); + rowCount += result.length; + LOG.info(result.length + " partitions are added. (total:" + rowCount + ")"); + lastIndex = i; + } } - if (stmt != null) { - stmt.executeBatch(); + if (lastIndex != i) { + int[] result = stmt.executeBatch(); + stmt.clearBatch(); + rowCount += result.length; + LOG.info(result.length + " partitions are added. (total:" + rowCount + ")"); + } + + lastIndex = 0; + rowCount = 0; + for(i = 0; i < partitions.size(); i++) { + PartitionDescProto partition = partitions.get(i); + + addPartitionKeys(tableId, partition, stmt, sb); + + if (i >= lastIndex + batchSize && lastIndex != i) { + int[] result = stmt.executeBatch(); + stmt.clearBatch(); + lastIndex = i; + rowCount += result.length; + LOG.info(result.length + " partition keys are added. (total:" + rowCount + ")"); + } + } + + if (lastIndex != i) { + int[] result = stmt.executeBatch(); + stmt.clearBatch(); + rowCount += result.length; + LOG.info(result.length + " partition keys are added. (total:" + rowCount + ")"); } if (conn != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index 1e25857c6f..a5e8647f78 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -35,6 +35,8 @@ import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.partition.PartitionDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.GlobalPlanner; @@ -310,7 +312,16 @@ public void stopQuery(QueryId queryId) { // If there is partitions if (stats.getPartitions() != null && stats.getPartitions().size() > 0) { - LOG.info("### Adding partitions START ###"); + + List partitions = TUtil.newList(); + for (CatalogProtos.PartitionDescProto partition: stats.getPartitions()) { + if (!partitions.contains(partition.getPartitionName())) { + partitions.add(partition.getPartitionName()); + } + } + + LOG.info("### Adding partitions START ###"); // Debug logs + CatalogService catalog = catalog = queryMasterContext.getWorkerContext().getCatalog(); // Store partitions to CatalogStore using alter table statement. boolean result = catalog.addPartitions(databaseName, simpleTableName, stats.getPartitions(), true); @@ -319,7 +330,8 @@ public void stopQuery(QueryId queryId) { } else { LOG.info(String.format("Incomplete adding for partition %s", stats.getPartitions().size())); } - LOG.info("### Adding partitions END ###"); + + LOG.info("### Adding partitions END ###"); // Debug logs } else { LOG.info("Can't find partitions for adding."); } From 305880181e3338761dcea2b6108b7219144e19c0 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 23 Jul 2015 18:11:20 +0900 Subject: [PATCH 21/40] Remove batchSize --- .../tajo/catalog/store/AbstractDBStore.java | 40 +------------------ 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 705f173c22..8784807ccc 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2192,8 +2192,7 @@ public void addPartitions(String databaseName, String tableName, List= lastIndex + batchSize && lastIndex != i) { - int[] result = stmt.executeBatch(); - stmt.clearBatch(); - rowCount += result.length; - LOG.info(result.length + " partitions are added. (total:" + rowCount + ")"); - lastIndex = i; - } - } - - if (lastIndex != i) { - int[] result = stmt.executeBatch(); - stmt.clearBatch(); - rowCount += result.length; - LOG.info(result.length + " partitions are added. (total:" + rowCount + ")"); - } - - lastIndex = 0; - rowCount = 0; - for(i = 0; i < partitions.size(); i++) { - PartitionDescProto partition = partitions.get(i); - addPartitionKeys(tableId, partition, stmt, sb); - - if (i >= lastIndex + batchSize && lastIndex != i) { - int[] result = stmt.executeBatch(); - stmt.clearBatch(); - lastIndex = i; - rowCount += result.length; - LOG.info(result.length + " partition keys are added. (total:" + rowCount + ")"); - } - } - - if (lastIndex != i) { - int[] result = stmt.executeBatch(); + stmt.executeBatch(); stmt.clearBatch(); - rowCount += result.length; - LOG.info(result.length + " partition keys are added. (total:" + rowCount + ")"); } if (conn != null) { From d8f09289f8c07c6dc3b4fc4a5be3ceb1b57afb75 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 24 Jul 2015 01:39:13 +0900 Subject: [PATCH 22/40] Add PARTITION_BULK_INSERT_BATCH_SIZE to configuration --- .../tajo/catalog/store/AbstractDBStore.java | 21 ++++++++++++------- .../java/org/apache/tajo/conf/TajoConf.java | 4 ++++ .../apache/tajo/querymaster/QueryMaster.java | 4 ---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 8784807ccc..52e060ee2f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -35,6 +35,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.InternalException; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.util.FileUtil; @@ -42,11 +43,7 @@ import org.apache.tajo.util.TUtil; import java.io.IOException; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.util.*; import static org.apache.tajo.catalog.exception.CatalogExceptionUtil.makeCatalogUpgrade; @@ -2192,7 +2189,9 @@ public void addPartitions(String databaseName, String tableName, List= lastIndex + batchSize && lastIndex != i) { + stmt.executeBatch(); + stmt.clearBatch(); + lastIndex = i; + } + } + + if (lastIndex != i) { stmt.executeBatch(); - stmt.clearBatch(); } if (conn != null) { diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index b8767376fa..2854531e11 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -290,6 +290,10 @@ public static enum ConfVars implements ConfigKey { PYTHON_CODE_DIR("tajo.function.python.code-dir", ""), PYTHON_CONTROLLER_LOG_DIR("tajo.function.python.controller.log-dir", ""), + // Partition + PARTITION_BULK_INSERT_BATCH_SIZE("tajo.partition.bulk.insert.batch.size", 1000), + + ///////////////////////////////////////////////////////////////////////////////// // User Session Configuration // diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index a5e8647f78..cf1f4add91 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -320,8 +320,6 @@ public void stopQuery(QueryId queryId) { } } - LOG.info("### Adding partitions START ###"); // Debug logs - CatalogService catalog = catalog = queryMasterContext.getWorkerContext().getCatalog(); // Store partitions to CatalogStore using alter table statement. boolean result = catalog.addPartitions(databaseName, simpleTableName, stats.getPartitions(), true); @@ -330,8 +328,6 @@ public void stopQuery(QueryId queryId) { } else { LOG.info(String.format("Incomplete adding for partition %s", stats.getPartitions().size())); } - - LOG.info("### Adding partitions END ###"); // Debug logs } else { LOG.info("Can't find partitions for adding."); } From 315d675ab389bfb4c3436298fe70f23002ec16f7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 24 Jul 2015 01:53:28 +0900 Subject: [PATCH 23/40] Remove unique constraint and add unique index --- .../src/main/resources/schemas/derby/derby.xml | 5 +++-- .../src/main/resources/schemas/mariadb/mariadb.xml | 4 ++-- .../src/main/resources/schemas/mysql/mysql.xml | 4 ++-- .../src/main/resources/schemas/oracle/oracle.xml | 7 +++++-- .../main/resources/schemas/postgresql/postgresql.xml | 12 +++++++----- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index e0bd469e8f..2661bc24ed 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -24,7 +24,7 @@ * 2 - 2014-06-09: First versioning * 1- Before 2013-03-20 --> - + @@ -157,7 +157,8 @@ TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE, PARTITION_NAME VARCHAR(767), PATH VARCHAR(1024), - CONSTRAINT C_PARTITION_PK PRIMARY KEY (PARTITION_ID) + CONSTRAINT C_PARTITIONS_PK PRIMARY KEY (PARTITION_ID), + CONSTRAINT C_PARTITIONS_UNIQ UNIQUE (TID, PARTITION_NAME) )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml index 7485da1f63..aeddf1b882 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml @@ -25,7 +25,7 @@ * 2 - 2014-06-09: First versioning * 1- Before 2013-03-20 --> - + @@ -140,7 +140,7 @@ TID INT NOT NULL, PARTITION_NAME VARCHAR(255) BINARY, PATH VARCHAR(4096) BINARY, - CONSTRAINT CONST_PARTITION_UNIQUE UNIQUE (PARTITION_NAME), + UNIQUE INDEX PARTITION_UNIQUE_IDX (TID, PARTITION_NAME), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml index 2bde04f440..ef72e4a2ba 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml @@ -25,7 +25,7 @@ * 2 - 2014-06-09: First versioning * 1- Before 2013-03-20 --> - + @@ -141,7 +141,7 @@ TID INT NOT NULL, PARTITION_NAME VARCHAR(255) BINARY, PATH VARCHAR(4096) BINARY, - CONSTRAINT CONST_PARTITION_UNIQUE UNIQUE (PARTITION_NAME), + UNIQUE INDEX PARTITION_UNIQUE_IDX (TID, PARTITION_NAME), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml index 2778e0cc10..30d15324b0 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml @@ -25,7 +25,7 @@ * 2 - 2014-06-09: First versioning * 1- Before 2013-03-20 --> - + - + + + + - + @@ -157,12 +157,14 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition. PARTITION_NAME VARCHAR(128), PARTITION_VALUE VARCHAR(1024), PATH VARCHAR(4096), - FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE, - CONSTRAINT C_PARTITION_UNIQUE UNIQUE (PARTITION_NAME) + FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE )]]> - + + + + - + From 832cdedfe8d9b0038bfc31bfb2e258e181740432 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 24 Jul 2015 09:50:19 +0900 Subject: [PATCH 24/40] Add trivial comment --- .../apache/tajo/catalog/store/AbstractDBStore.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 52e060ee2f..306ddaf22e 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -2189,10 +2189,12 @@ public void addPartitions(String databaseName, String tableName, List= lastIndex + batchSize && lastIndex != i) { + if (currentIndex >= lastIndex + batchSize && lastIndex != currentIndex) { stmt.executeBatch(); stmt.clearBatch(); - lastIndex = i; + lastIndex = currentIndex; } } - if (lastIndex != i) { + if (lastIndex != currentIndex) { stmt.executeBatch(); } From 30687af0d6104a6ca9192438ab8b6db5df58cab7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 27 Jul 2015 12:51:06 +0900 Subject: [PATCH 25/40] Remove unnecessary packages --- .../tajo/engine/planner/physical/ColPartitionStoreExec.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index 571ec8f520..b79a8180f2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -38,12 +38,9 @@ import org.apache.tajo.plan.logical.StoreTableNode; import org.apache.tajo.storage.*; import org.apache.tajo.unit.StorageUnit; -import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; -import java.net.URI; -import java.util.List; public abstract class ColPartitionStoreExec extends UnaryPhysicalExec { private static Log LOG = LogFactory.getLog(ColPartitionStoreExec.class); From d9bddaada26e9111ea898f6a696541c8c76f4ef2 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 27 Jul 2015 12:58:13 +0900 Subject: [PATCH 26/40] Add update history for catalog schema --- .../src/main/resources/schemas/derby/derby.xml | 1 + .../src/main/resources/schemas/mariadb/mariadb.xml | 1 + .../src/main/resources/schemas/mysql/mysql.xml | 1 + .../src/main/resources/schemas/oracle/oracle.xml | 1 + .../src/main/resources/schemas/postgresql/postgresql.xml | 1 + 5 files changed, 5 insertions(+) diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index 2661bc24ed..ed803031a0 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -19,6 +19,7 @@ @@ -61,7 +61,7 @@ tajo.catalog.jdbc.uri - jdbc:mariadb://:/?createDatabaseIfNotExist=true + jdbc:mariadb://:/?rewriteBatchedStatements=true --> From 2a3be8e03be3d1ea884b86a87423a6300cdd473e Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 30 Jul 2015 23:05:35 +0900 Subject: [PATCH 35/40] Remove parent column name from partition_keys table --- .../org/apache/tajo/catalog/CatalogConstants.java | 1 - .../java/org/apache/tajo/catalog/CatalogUtil.java | 5 ----- .../apache/tajo/catalog/store/AbstractDBStore.java | 13 +++++-------- .../org/apache/tajo/catalog/store/MemStore.java | 1 - .../src/main/resources/schemas/derby/derby.xml | 3 +-- .../src/main/resources/schemas/mariadb/mariadb.xml | 3 +-- .../src/main/resources/schemas/mysql/mysql.xml | 3 +-- .../src/main/resources/schemas/oracle/oracle.xml | 3 +-- .../resources/schemas/postgresql/postgresql.xml | 3 +-- .../java/org/apache/tajo/catalog/TestCatalog.java | 8 -------- .../planner/physical/ColPartitionStoreExec.java | 10 ++-------- .../tajo/engine/query/TestTablePartitions.java | 8 -------- 12 files changed, 12 insertions(+), 49 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java index 475c0c7f77..721bcf1003 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java @@ -58,7 +58,6 @@ public class CatalogConstants { public static final String COL_PARTITIONS_PK = "PARTITION_ID"; public static final String COL_COLUMN_NAME = "COLUMN_NAME"; - public static final String COL_PARENT_COLUMN_NAME = "PARENT_COLUMN_NAME"; public static final String COL_PARTITION_VALUE = "PARTITION_VALUE"; public static final String INFORMATION_SCHEMA_DB_NAME = "information_schema"; diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 8f3a83d25b..378bbcb2df 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -851,11 +851,6 @@ public static Pair, String> getPartitionKeyNamePair(Stri for (int i = 0; i < columns.length; i++) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(columns[i]); - - if (i > 0) { - builder.setParentColumnName(columns[i-1]); - } - builder.setPartitionValue(values[i]); if (i > 0) { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 862737b0f9..9dc60bdd3d 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -61,11 +61,11 @@ public abstract class AbstractDBStore extends CatalogConstants implements Catalo protected final String insertPartitionKeysSql = "INSERT INTO " + TB_PARTTION_KEYS + "(" + COL_PARTITIONS_PK + ", " + COL_TABLES_PK + ", " - + COL_COLUMN_NAME + ", " + COL_PARENT_COLUMN_NAME + ", " + COL_PARTITION_VALUE + ")" + + COL_COLUMN_NAME + ", " + COL_PARTITION_VALUE + ")" + " VALUES ( (" + " SELECT " + COL_PARTITIONS_PK + " FROM " + TB_PARTTIONS + " WHERE " + COL_TABLES_PK + " = ? AND PARTITION_NAME = ? ) " - + " , ?, ?, ? , ?)"; + + " , ?, ?, ?)"; protected final String deletePartitionSql = "DELETE FROM " + TB_PARTTIONS + " WHERE " + COL_PARTITIONS_PK + " = ? "; @@ -1277,8 +1277,7 @@ public void addPartition(int tableId, CatalogProtos.PartitionDescProto partition pstmt2.setString(2, partition.getPartitionName()); pstmt2.setInt(3, tableId); pstmt2.setString(4, partitionKey.getColumnName()); - pstmt2.setString(5, partitionKey.getParentColumnName()); - pstmt2.setString(6, partitionKey.getPartitionValue()); + pstmt2.setString(5, partitionKey.getPartitionValue()); pstmt2.addBatch(); pstmt2.clearParameters(); } @@ -2056,7 +2055,7 @@ private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) PreparedStatement pstmt = null; try { - String sql = "SELECT "+ COL_COLUMN_NAME + ", " + COL_PARENT_COLUMN_NAME + " , "+ COL_PARTITION_VALUE + String sql = "SELECT "+ COL_COLUMN_NAME + " , "+ COL_PARTITION_VALUE + " FROM " + TB_PARTTION_KEYS + " WHERE " + COL_PARTITIONS_PK + " = ? "; conn = getConnection(); @@ -2067,7 +2066,6 @@ private void setPartitionKeys(int pid, PartitionDescProto.Builder partitionDesc) while (res.next()) { PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); builder.setColumnName(res.getString(COL_COLUMN_NAME)); - builder.setParentColumnName(res.getString(COL_PARENT_COLUMN_NAME)); builder.setPartitionValue(res.getString(COL_PARTITION_VALUE)); partitionDesc.addPartitionKeys(builder); } @@ -2216,8 +2214,7 @@ public void addPartitions(String databaseName, String tableName, List