From 02a3c34008bcb40210746d226034e15fa3eff63e Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Sat, 27 Jun 2015 15:41:02 +0900 Subject: [PATCH 01/25] TAJO-1345: Implement logical plan part and DDL executor for alter partition. --- .../org/apache/tajo/catalog/CatalogUtil.java | 36 ++++++ .../apache/tajo/catalog/store/MemStore.java | 8 +- .../apache/tajo/master/exec/DDLExecutor.java | 25 +++++ .../engine/planner/TestLogicalPlanner.java | 103 ++++++++++++++++++ .../tajo/engine/query/TestAlterTable.java | 40 +++++++ .../org/apache/tajo/jdbc/TestTajoJdbc.java | 13 +-- .../alter_table_add_partition1.sql | 1 + .../alter_table_drop_partition1.sql | 1 + .../create_partitioned_table.sql | 1 + .../testAlterTableAddPartition.result | 2 +- .../testAlterTableDropPartition.result | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 49 +++++++++ .../tajo/plan/logical/AlterTableNode.java | 53 +++++++-- .../plan/serder/LogicalNodeDeserializer.java | 19 ++++ .../plan/serder/LogicalNodeSerializer.java | 28 +++++ .../plan/verifier/PreLogicalPlanVerifier.java | 6 - tajo-plan/src/main/proto/Plan.proto | 9 ++ 17 files changed, 367 insertions(+), 29 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition1.sql create mode 100644 tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition1.sql create mode 100644 tajo-core/src/test/resources/queries/TestAlterTable/create_partitioned_table.sql diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 6c6915b4cd..f7fb7121ea 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -23,6 +23,8 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.DataTypeUtil; import org.apache.tajo.TajoConstants; +import org.apache.tajo.catalog.partition.PartitionDesc; +import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; @@ -792,6 +794,40 @@ public static AlterTableDesc setProperty(String tableName, KeyValueSet params, A return alterTableDesc; } + public static AlterTableDesc addPartitionAndDropPartition(String tableName, String[] columns, + String[] values, String location, AlterTableType alterTableType) { + final AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(tableName); + + PartitionDesc partitionDesc = new PartitionDesc(); + + List partitionKeyList = TUtil.newList(); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < columns.length; i++) { + PartitionKey partitionKey = new PartitionKey(); + partitionKey.setColumnName(columns[i]); + partitionKey.setPartitionValue(values[i]); + + if (i > 0) { + sb.append("/"); + } + sb.append(columns[i]).append("=").append(values[i]); + partitionKeyList.add(partitionKey); + } + + partitionDesc.setPartitionKeys(partitionKeyList); + partitionDesc.setPartitionName(sb.toString()); + + if (alterTableType.equals(AlterTableType.ADD_PARTITION) && location != null) { + partitionDesc.setPath(location); + } + + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(alterTableType); + return alterTableDesc; + } + /* It is the relationship graph of type conversions. */ public static final Map> OPERATION_CASTING_MAP = Maps.newHashMap(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 55c43367a4..77950cbbba 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -322,13 +322,11 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th builder.setPath(partitionDesc.getPath()); if (partitionDesc.getPartitionKeysCount() > 0) { - int i = 0; for (CatalogProtos.PartitionKeyProto eachKey : partitionDesc.getPartitionKeysList()) { CatalogProtos.PartitionKeyProto.Builder keyBuilder = CatalogProtos.PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(eachKey.getColumnName()); keyBuilder.setPartitionValue(eachKey.getPartitionValue()); - builder.setPartitionKeys(i, keyBuilder.build()); - i++; + builder.addPartitionKeys(keyBuilder.build()); } } @@ -348,7 +346,9 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th if(!partitions.containsKey(tableName)) { throw new NoSuchPartitionException(databaseName, tableName, partitionName); } else { - partitions.remove(partitionName); + Map protoMap = partitions.get(tableName); + protoMap.remove(partitionName); + partitions.put(tableName, protoMap); } break; case SET_PROPERTY: diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 7104412db2..e0af0ff15a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -28,6 +28,7 @@ import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.*; +import org.apache.tajo.catalog.exception.NoSuchColumnException; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import org.apache.tajo.conf.TajoConf; @@ -446,11 +447,35 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer case SET_PROPERTY: catalog.alterTable(CatalogUtil.setProperty(qualifiedName, alterTable.getProperties(), AlterTableType.SET_PROPERTY)); break; + case ADD_PARTITION: + existColumnNames(qualifiedName, alterTable.getColumnNames()); + catalog.alterTable(CatalogUtil.addPartitionAndDropPartition(qualifiedName, alterTable.getColumnNames(), + alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION)); + break; + case DROP_PARTITION: + existColumnNames(qualifiedName, alterTable.getColumnNames()); + catalog.alterTable(CatalogUtil.addPartitionAndDropPartition(qualifiedName, alterTable.getColumnNames(), + alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.DROP_PARTITION)); + break; default: //TODO } } + private boolean existColumnNames(String tableName, String[] columnNames) { + for(String columnName : columnNames) { + if (!existPartitionColumnName(tableName, columnName)) { + throw new NoSuchColumnException(columnName); + } + } + return true; + } + + private boolean existPartitionColumnName(String tableName, String columnName) { + final TableDesc tableDesc = catalog.getTableDesc(tableName); + return tableDesc.getPartitionMethod().getExpressionSchema().contains(columnName) ? true : false; + } + private boolean existColumnName(String tableName, String columnName) { final TableDesc tableDesc = catalog.getTableDesc(tableName); return tableDesc.getSchema().containsByName(columnName) ? true : false; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 0f377630de..2b3df6588f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -25,10 +25,13 @@ import org.apache.tajo.QueryVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; +import org.apache.tajo.algebra.AlterTableOpType; import org.apache.tajo.algebra.Expr; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.benchmark.TPCH; import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.TextDatum; @@ -1230,4 +1233,104 @@ private static InsertNode getInsertNode(LogicalPlan plan) { assertEquals(NodeType.INSERT, root.getChild().getType()); return root.getChild(); } + + + String [] ALTER_PARTITIONS = { + "ALTER TABLE partitioned_table ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx" + + ".com/warehouse/partitioned_table/col1=1/col2=2'", //0 + "ALTER TABLE partitioned_table DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' )", //1 + }; + + @Test + public final void testAddPartitionAndDropPartition() throws PlanningException { + String tableName = CatalogUtil.normalizeIdentifier("partitioned_table"); + String qualifiedTableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName); + + Schema schema = new Schema(); + schema.addColumn("id", Type.INT4) + .addColumn("name", Type.TEXT) + .addColumn("age", Type.INT4) + .addColumn("score", Type.FLOAT8); + + KeyValueSet opts = new KeyValueSet(); + opts.set("file.delimiter", ","); + + Schema partSchema = new Schema(); + partSchema.addColumn("id", Type.INT4); + partSchema.addColumn("name", Type.TEXT); + + PartitionMethodDesc partitionMethodDesc = + new PartitionMethodDesc(DEFAULT_DATABASE_NAME, tableName, + CatalogProtos.PartitionType.COLUMN, "id,name", partSchema); + + TableDesc desc = null; + + try { + desc = new TableDesc(qualifiedTableName, schema, "CSV", new KeyValueSet(), + CommonTestingUtil.getTestDir().toUri()); + } catch (Exception e) { + throw new PlanningException(e.getMessage()); + } + + desc.setPartitionMethod(partitionMethodDesc); + assertFalse(catalog.existsTable(qualifiedTableName)); + catalog.createTable(desc); + assertTrue(catalog.existsTable(qualifiedTableName)); + + TableDesc retrieved = catalog.getTableDesc(qualifiedTableName); + assertEquals(retrieved.getName(), qualifiedTableName); + assertEquals(retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN); + assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id"); + + QueryContext qc = new QueryContext(util.getConfiguration(), session); + + // Testing alter table add partition + Expr expr = sqlAnalyzer.parse(ALTER_PARTITIONS[0]); + LogicalPlan rootNode = planner.createPlan(qc, expr); + LogicalNode plan = rootNode.getRootBlock().getRoot(); + testJsonSerDerObject(plan); + assertEquals(NodeType.ROOT, plan.getType()); + LogicalRootNode root = (LogicalRootNode) plan; + assertEquals(NodeType.ALTER_TABLE, root.getChild().getType()); + + AlterTableNode alterTableNode = root.getChild(); + + assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); + + assertEquals(alterTableNode.getColumnNames().length, 2); + assertEquals(alterTableNode.getPartitionValues().length, 2); + + assertEquals(alterTableNode.getColumnNames()[0], "col1"); + assertEquals(alterTableNode.getColumnNames()[1], "col2"); + + assertEquals(alterTableNode.getPartitionValues()[0], "1"); + assertEquals(alterTableNode.getPartitionValues()[1], "2"); + + assertEquals(alterTableNode.getLocation(), "hdfs://xxx.com/warehouse/partitioned_table/col1=1/col2=2"); + + // Testing alter table drop partition + expr = sqlAnalyzer.parse(ALTER_PARTITIONS[1]); + rootNode = planner.createPlan(qc, expr); + plan = rootNode.getRootBlock().getRoot(); + testJsonSerDerObject(plan); + assertEquals(NodeType.ROOT, plan.getType()); + root = (LogicalRootNode) plan; + assertEquals(NodeType.ALTER_TABLE, root.getChild().getType()); + + alterTableNode = root.getChild(); + + assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.DROP_PARTITION); + + assertEquals(alterTableNode.getColumnNames().length, 3); + assertEquals(alterTableNode.getPartitionValues().length, 3); + + assertEquals(alterTableNode.getColumnNames()[0], "col1"); + assertEquals(alterTableNode.getColumnNames()[1], "col2"); + assertEquals(alterTableNode.getColumnNames()[2], "col3"); + + assertEquals(alterTableNode.getPartitionValues()[0], "2015"); + assertEquals(alterTableNode.getPartitionValues()[1], "01"); + assertEquals(alterTableNode.getPartitionValues()[2], "11"); + } + } \ No newline at end of file diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 44fa1f3594..c692c24b3c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -20,12 +20,20 @@ import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.junit.Test; import org.junit.experimental.categories.Category; import java.sql.ResultSet; import java.util.List; +import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + @Category(IntegrationTest.class) public class TestAlterTable extends QueryTestCaseBase { @Test @@ -63,4 +71,36 @@ public final void testAlterTableSetProperty() throws Exception { assertResultSet(after_res, "after_set_property_delimiter.result"); cleanupQuery(after_res); } + + @Test + public final void testAlterTableAddPartition() throws Exception { + executeDDL("create_partitioned_table.sql", null); + + String tableName = CatalogUtil.buildFQName("TestAlterTable", "partitioned_table"); + assertTrue(catalog.existsTable(tableName)); + + TableDesc retrieved = catalog.getTableDesc(tableName); + assertEquals(retrieved.getName(), tableName); + assertEquals(retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN); + assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getAllColumns().size(), 2); + assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "col3"); + assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(1).getSimpleName(), "col4"); + + executeDDL("alter_table_add_partition1.sql", null); + + List partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + assertNotNull(partitions); + assertEquals(partitions.size(), 1); + assertEquals(partitions.get(0).getPartitionName(), "col3=1/col4=2"); + assertEquals(partitions.get(0).getPartitionKeysList().get(0).getColumnName(), "col3"); + assertEquals(partitions.get(0).getPartitionKeysList().get(0).getPartitionValue(), "1"); + assertEquals(partitions.get(0).getPartitionKeysList().get(1).getColumnName(), "col4"); + assertEquals(partitions.get(0).getPartitionKeysList().get(1).getPartitionValue(), "2"); + + executeDDL("alter_table_drop_partition1.sql", null); + + partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); + assertNotNull(partitions); + assertEquals(partitions.size(), 0); + } } diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index ad74046712..0f49bc1885 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -604,8 +604,9 @@ public void testAlterTableAddPartition() throws Exception { // It is because HiveCatalogStore does not support Time data type. try { if (!testingCluster.isHiveCatalogStoreRunning()) { - String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), - DEFAULT_DATABASE_NAME); + String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), + tajoMasterAddress.getPort(), "TestTajoJdbc"); + conn = DriverManager.getConnection(connUri); assertTrue(conn.isValid(100)); @@ -615,12 +616,10 @@ public void testAlterTableAddPartition() throws Exception { resultSet.close(); stmt = conn.createStatement(); - resultSet = stmt.executeQuery("alter table " + tableName + " add partition (key = 0.1)"); - } - } catch (SQLException e) { - errorMessage = e.getMessage(); + result = stmt.executeUpdate("alter table " + tableName + " add partition (key = 0.1)"); + assertEquals(result, 1); + } } finally { - assertEquals(errorMessage, "ADD_PARTITION is not supported yet\n"); cleanupQuery(resultSet); if (stmt != null) { stmt.close(); diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition1.sql b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition1.sql new file mode 100644 index 0000000000..11546835bf --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_add_partition1.sql @@ -0,0 +1 @@ +ALTER TABLE partitioned_table ADD PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition1.sql b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition1.sql new file mode 100644 index 0000000000..b5d672fc52 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/alter_table_drop_partition1.sql @@ -0,0 +1 @@ +ALTER TABLE partitioned_table DROP PARTITION (col3 = 1 , col4 = 2) \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestAlterTable/create_partitioned_table.sql b/tajo-core/src/test/resources/queries/TestAlterTable/create_partitioned_table.sql new file mode 100644 index 0000000000..b349349a46 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestAlterTable/create_partitioned_table.sql @@ -0,0 +1 @@ +create table partitioned_table (col1 int4, col2 int4) partition by column(col3 int4, col4 int4) \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result index 31f46bc4f3..bcaba86266 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result @@ -1,2 +1,2 @@ OK -ERROR: ADD_PARTITION is not supported yet \ No newline at end of file +OK \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result index 1fadcea500..bcaba86266 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result @@ -1,2 +1,2 @@ OK -ERROR: DROP_PARTITION is not supported yet \ No newline at end of file +OK \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index a2480c9f6b..74dd849ec3 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1941,6 +1941,42 @@ private static Schema convertTableElementsSchema(ColumnDefinition[] elements) { return schema; } + /** + * It transforms ColumnDefinition array to String array. + * + * @param columnReferenceExprs + * @return + */ + private static String[] convertColumnsToStrings(ColumnReferenceExpr[] columnReferenceExprs) { + int columnCount = columnReferenceExprs.length; + String[] columns = new String[columnCount]; + + for(int i = 0; i < columnCount; i++) { + ColumnReferenceExpr columnReferenceExpr = columnReferenceExprs[i]; + columns[i] = columnReferenceExpr.getName(); + } + + return columns; + } + + /** + * It transforms Expr array to String array. + * + * @param exprs + * @return + */ + private static String[] convertExprsToStrings(Expr[] exprs) { + int exprCount = exprs.length; + String[] values = new String[exprCount]; + + for(int i = 0; i < exprCount; i++) { + LiteralValue expr = (LiteralValue)exprs[i]; + values[i] = expr.getValue(); + } + + return values; + } + private static Column convertColumn(ColumnDefinition columnDefinition) { return new Column(columnDefinition.getColumnName(), convertDataType(columnDefinition)); } @@ -2004,6 +2040,19 @@ public LogicalNode visitAlterTable(PlanContext context, Stack stack, Alter if (null != alterTable.getAddNewColumn()) { alterTableNode.setAddNewColumn(convertColumn(alterTable.getAddNewColumn())); } + + if (alterTable.getColumns() != null) { + alterTableNode.setColumnNames(convertColumnsToStrings(alterTable.getColumns())); + } + + if (alterTable.getValues() != null) { + alterTableNode.setPartitionValues(convertExprsToStrings(alterTable.getValues())); + } + + if (alterTable.getLocation() != null) { + alterTableNode.setLocation(alterTable.getLocation()); + } + alterTableNode.setAlterTableOpType(alterTable.getAlterTableOpType()); return alterTableNode; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java index 9cbde0ce06..7a214f3037 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java @@ -19,6 +19,7 @@ package org.apache.tajo.plan.logical; +import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.algebra.AlterTableOpType; @@ -42,6 +43,12 @@ public class AlterTableNode extends LogicalNode { private KeyValueSet properties = new KeyValueSet(); @Expose private AlterTableOpType alterTableOpType; + @Expose + private String[] columnNames; + @Expose + private String[] partitionValues; + @Expose + private String location; public AlterTableNode(int pid) { super(pid, NodeType.ALTER_TABLE); @@ -117,6 +124,30 @@ public void setProperties(KeyValueSet properties) { this.properties = properties; } + public String[] getColumnNames() { + return columnNames; + } + + public void setColumnNames(String[] columnNames) { + this.columnNames = columnNames; + } + + public String[] getPartitionValues() { + return partitionValues; + } + + public void setPartitionValues(String[] partitionValues) { + this.partitionValues = partitionValues; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + @Override public PlanString getPlanString() { return new PlanString(this); @@ -124,16 +155,18 @@ public PlanString getPlanString() { @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((addNewColumn == null) ? 0 : addNewColumn.hashCode()); - result = prime * result + ((alterTableOpType == null) ? 0 : alterTableOpType.hashCode()); - result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()); - result = prime * result + ((newColumnName == null) ? 0 : newColumnName.hashCode()); - result = prime * result + ((newTableName == null) ? 0 : newTableName.hashCode()); - result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); - result = prime * result + ((properties == null) ? 0 : properties.hashCode()); - return result; + return Objects.hashCode(tableName, + null != addNewColumn ? Objects.hashCode(addNewColumn) : addNewColumn, + null != alterTableOpType ? Objects.hashCode(alterTableOpType) : alterTableOpType, + null != columnName ? Objects.hashCode(columnName) : columnName, + null != newColumnName ? Objects.hashCode(newColumnName) : newColumnName, + null != newTableName ? Objects.hashCode(newTableName) : newTableName, + null != tableName ? Objects.hashCode(tableName) : tableName, + null != properties ? Objects.hashCode(properties) : properties, + null != columnNames ? Objects.hashCode(columnNames) : columnNames, + null != partitionValues ? Objects.hashCode(partitionValues) : partitionValues, + null != location ? Objects.hashCode(location) : location + ); } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index c1d9f9a11b..5ab3bad9a9 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -579,6 +579,8 @@ private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) PlanProto.AlterTableNode alterTableProto = protoNode.getAlterTable(); alterTable.setTableName(alterTableProto.getTableName()); + PlanProto.AlterTableNode.AlterPartition alterPartition = null; + switch (alterTableProto.getSetType()) { case RENAME_TABLE: alterTable.setNewTableName(alterTableProto.getRenameTable().getNewName()); @@ -593,6 +595,23 @@ private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) case SET_PROPERTY: alterTable.setProperties(new KeyValueSet(alterTableProto.getProperties())); break; + case ADD_PARTITION: + alterPartition = alterTableProto.getAlterPartition(); + alterTable.setColumnNames(alterPartition.getColumnNamesList().toArray(new String[alterPartition + .getColumnNamesCount()])); + alterTable.setPartitionValues(alterPartition.getPartitionValuesList().toArray(new String[alterPartition + .getPartitionValuesCount()])); + if (alterPartition.getLocation() != null) { + alterTable.setLocation(alterPartition.getLocation()); + } + break; + case DROP_PARTITION: + alterPartition = alterTableProto.getAlterPartition(); + alterTable.setColumnNames(alterPartition.getColumnNamesList().toArray(new String[alterPartition + .getColumnNamesCount()])); + alterTable.setPartitionValues(alterPartition.getPartitionValuesList().toArray(new String[alterPartition + .getPartitionValuesCount()])); + break; default: throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + alterTableProto.getSetType().name()); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index 6737756c65..e18276a4aa 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -549,6 +549,7 @@ public LogicalNode visitAlterTable(SerializeContext context, LogicalPlan plan, L AlterTableNode node, Stack stack) { PlanProto.AlterTableNode.Builder alterTableBuilder = PlanProto.AlterTableNode.newBuilder(); alterTableBuilder.setTableName(node.getTableName()); + PlanProto.AlterTableNode.AlterPartition.Builder partitionBuilder = null; switch (node.getAlterTableOpType()) { case RENAME_TABLE: @@ -569,6 +570,33 @@ public LogicalNode visitAlterTable(SerializeContext context, LogicalPlan plan, L alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.SET_PROPERTY); alterTableBuilder.setProperties(node.getProperties().getProto()); break; + case ADD_PARTITION: + alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.ADD_PARTITION); + partitionBuilder = PlanProto.AlterTableNode.AlterPartition.newBuilder(); + for (String columnName : node.getColumnNames()) { + partitionBuilder.addColumnNames(columnName); + } + + for (String partitionValue : node.getPartitionValues()) { + partitionBuilder.addPartitionValues(partitionValue); + } + if (node.getLocation() != null) { + partitionBuilder.setLocation(node.getLocation()); + } + alterTableBuilder.setAlterPartition(partitionBuilder); + break; + case DROP_PARTITION: + alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.DROP_PARTITION); + partitionBuilder = PlanProto.AlterTableNode.AlterPartition.newBuilder(); + for (String columnName : node.getColumnNames()) { + partitionBuilder.addColumnNames(columnName); + } + + for (String partitionValue : node.getPartitionValues()) { + partitionBuilder.addPartitionValues(partitionValue); + } + alterTableBuilder.setAlterPartition(partitionBuilder); + break; default: throw new UnimplementedException("Unknown SET type in ALTER TABLE: " + node.getAlterTableOpType().name()); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 09160abc1d..18ca627f22 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -320,12 +320,6 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws @Override public Expr visitAlterTable(Context context, Stack stack, AlterTable expr) throws PlanningException { super.visitAlterTable(context, stack, expr); - - if (expr.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION - || expr.getAlterTableOpType() == AlterTableOpType.DROP_PARTITION) { - context.state.addVerification(expr.getAlterTableOpType().name() + " is not supported yet"); - } - return expr; } } diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 40b789118c..25f51d699f 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -280,6 +280,8 @@ message AlterTableNode { RENAME_COLUMN = 1; ADD_COLUMN = 2; SET_PROPERTY = 3; + ADD_PARTITION = 4; + DROP_PARTITION = 5; } message RenameTable { @@ -295,12 +297,19 @@ message AlterTableNode { required ColumnProto addColumn = 1; } + message AlterPartition { + repeated string columnNames = 1; + repeated string partitionValues = 21; + optional string location = 3; + } + required string tableName = 1; required Type setType = 2; optional RenameTable renameTable = 3; optional RenameColumn renameColumn = 4; optional AddColumn addColumn = 5; optional KeyValueSetProto properties = 6; + optional AlterPartition alterPartition = 7; } enum EvalType { From 3e153a01c80bd3828ab0d1d2e2e57e5526e56e60 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Sun, 28 Jun 2015 00:57:38 +0900 Subject: [PATCH 02/25] Trigger for travis build. --- .../java/org/apache/tajo/engine/planner/TestLogicalPlanner.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 2b3df6588f..8626418dd4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -1234,7 +1234,6 @@ private static InsertNode getInsertNode(LogicalPlan plan) { return root.getChild(); } - String [] ALTER_PARTITIONS = { "ALTER TABLE partitioned_table ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx" + ".com/warehouse/partitioned_table/col1=1/col2=2'", //0 From d797a85072e443a65157f98275fea3640a3541a7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 3 Jul 2015 18:06:06 +0900 Subject: [PATCH 03/25] Remove unnecessary codes. --- .../org/apache/tajo/catalog/CatalogUtil.java | 2 +- .../NoSuchPartitionKeyException.java | 39 +++++++++++++++++++ .../apache/tajo/catalog/store/MemStore.java | 4 +- .../apache/tajo/master/exec/DDLExecutor.java | 18 +++++---- .../engine/planner/TestLogicalPlanner.java | 14 +++---- .../org/apache/tajo/plan/LogicalPlanner.java | 2 +- .../tajo/plan/logical/AlterTableNode.java | 24 ++++-------- .../plan/serder/LogicalNodeDeserializer.java | 4 +- .../plan/serder/LogicalNodeSerializer.java | 4 +- 9 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index b7ca868838..7e8a63f1e2 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -794,7 +794,7 @@ public static AlterTableDesc setProperty(String tableName, KeyValueSet params, A return alterTableDesc; } - public static AlterTableDesc addPartitionAndDropPartition(String tableName, String[] columns, + public static AlterTableDesc addOrDropPartition(String tableName, String[] columns, String[] values, String location, AlterTableType alterTableType) { final AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(tableName); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java new file mode 100644 index 0000000000..bc56580655 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog.exception; + +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.function.FunctionUtil; +import org.codehaus.jackson.schema.JsonSerializableSchema; + +import java.util.Collection; + +public class NoSuchPartitionKeyException extends RuntimeException { + + private static final long serialVersionUID = 277182608283894939L; + + public NoSuchPartitionKeyException(String message) { + super(message); + } + + public NoSuchPartitionKeyException(String databaseName, String tableName, String partitionKey) { + super(String.format("ERROR: \"%s\" does not exist in \"%s.%s\".", partitionKey, databaseName, tableName)); + } + +} diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 77950cbbba..8f1ac952ba 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -346,9 +346,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) th if(!partitions.containsKey(tableName)) { throw new NoSuchPartitionException(databaseName, tableName, partitionName); } else { - Map protoMap = partitions.get(tableName); - protoMap.remove(partitionName); - partitions.put(tableName, protoMap); + partitions.get(tableName).remove(partitionName); } break; case SET_PROPERTY: diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index e0af0ff15a..f81bad22b9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -448,13 +448,13 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer catalog.alterTable(CatalogUtil.setProperty(qualifiedName, alterTable.getProperties(), AlterTableType.SET_PROPERTY)); break; case ADD_PARTITION: - existColumnNames(qualifiedName, alterTable.getColumnNames()); - catalog.alterTable(CatalogUtil.addPartitionAndDropPartition(qualifiedName, alterTable.getColumnNames(), + existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION)); break; case DROP_PARTITION: - existColumnNames(qualifiedName, alterTable.getColumnNames()); - catalog.alterTable(CatalogUtil.addPartitionAndDropPartition(qualifiedName, alterTable.getColumnNames(), + existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.DROP_PARTITION)); break; default: @@ -462,10 +462,10 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer } } - private boolean existColumnNames(String tableName, String[] columnNames) { + private boolean existPartitionColumnNames(String tableName, String[] columnNames) { for(String columnName : columnNames) { if (!existPartitionColumnName(tableName, columnName)) { - throw new NoSuchColumnException(columnName); + throw new NoSuchPartitionKeyException(columnName); } } return true; @@ -473,7 +473,11 @@ private boolean existColumnNames(String tableName, String[] columnNames) { private boolean existPartitionColumnName(String tableName, String columnName) { final TableDesc tableDesc = catalog.getTableDesc(tableName); - return tableDesc.getPartitionMethod().getExpressionSchema().contains(columnName) ? true : false; + if (tableDesc.getPartitionMethod().getExpressionSchema().contains(columnName)) { + return true; + } else { + return false; + } } private boolean existColumnName(String tableName, String columnName) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 8626418dd4..741c1acaf5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -1296,11 +1296,11 @@ public final void testAddPartitionAndDropPartition() throws PlanningException { assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); - assertEquals(alterTableNode.getColumnNames().length, 2); + assertEquals(alterTableNode.getPartitionColumns().length, 2); assertEquals(alterTableNode.getPartitionValues().length, 2); - assertEquals(alterTableNode.getColumnNames()[0], "col1"); - assertEquals(alterTableNode.getColumnNames()[1], "col2"); + assertEquals(alterTableNode.getPartitionColumns()[0], "col1"); + assertEquals(alterTableNode.getPartitionColumns()[1], "col2"); assertEquals(alterTableNode.getPartitionValues()[0], "1"); assertEquals(alterTableNode.getPartitionValues()[1], "2"); @@ -1320,12 +1320,12 @@ public final void testAddPartitionAndDropPartition() throws PlanningException { assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.DROP_PARTITION); - assertEquals(alterTableNode.getColumnNames().length, 3); + assertEquals(alterTableNode.getPartitionColumns().length, 3); assertEquals(alterTableNode.getPartitionValues().length, 3); - assertEquals(alterTableNode.getColumnNames()[0], "col1"); - assertEquals(alterTableNode.getColumnNames()[1], "col2"); - assertEquals(alterTableNode.getColumnNames()[2], "col3"); + assertEquals(alterTableNode.getPartitionColumns()[0], "col1"); + assertEquals(alterTableNode.getPartitionColumns()[1], "col2"); + assertEquals(alterTableNode.getPartitionColumns()[2], "col3"); assertEquals(alterTableNode.getPartitionValues()[0], "2015"); assertEquals(alterTableNode.getPartitionValues()[1], "01"); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 10423265de..bdc8d81486 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -2042,7 +2042,7 @@ public LogicalNode visitAlterTable(PlanContext context, Stack stack, Alter } if (alterTable.getColumns() != null) { - alterTableNode.setColumnNames(convertColumnsToStrings(alterTable.getColumns())); + alterTableNode.setPartitionColumns(convertColumnsToStrings(alterTable.getColumns())); } if (alterTable.getValues() != null) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java index 7a214f3037..8459aeb436 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java @@ -44,7 +44,7 @@ public class AlterTableNode extends LogicalNode { @Expose private AlterTableOpType alterTableOpType; @Expose - private String[] columnNames; + private String[] partitionColumns; @Expose private String[] partitionValues; @Expose @@ -124,12 +124,12 @@ public void setProperties(KeyValueSet properties) { this.properties = properties; } - public String[] getColumnNames() { - return columnNames; + public String[] getPartitionColumns() { + return partitionColumns; } - public void setColumnNames(String[] columnNames) { - this.columnNames = columnNames; + public void setPartitionColumns(String[] partitionColumns) { + this.partitionColumns = partitionColumns; } public String[] getPartitionValues() { @@ -155,18 +155,8 @@ public PlanString getPlanString() { @Override public int hashCode() { - return Objects.hashCode(tableName, - null != addNewColumn ? Objects.hashCode(addNewColumn) : addNewColumn, - null != alterTableOpType ? Objects.hashCode(alterTableOpType) : alterTableOpType, - null != columnName ? Objects.hashCode(columnName) : columnName, - null != newColumnName ? Objects.hashCode(newColumnName) : newColumnName, - null != newTableName ? Objects.hashCode(newTableName) : newTableName, - null != tableName ? Objects.hashCode(tableName) : tableName, - null != properties ? Objects.hashCode(properties) : properties, - null != columnNames ? Objects.hashCode(columnNames) : columnNames, - null != partitionValues ? Objects.hashCode(partitionValues) : partitionValues, - null != location ? Objects.hashCode(location) : location - ); + return Objects.hashCode(tableName, addNewColumn, alterTableOpType, columnName, newColumnName, newTableName, + tableName, properties, partitionColumns, partitionValues, location); } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java index 5ab3bad9a9..fe900c05ca 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeDeserializer.java @@ -597,7 +597,7 @@ private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) break; case ADD_PARTITION: alterPartition = alterTableProto.getAlterPartition(); - alterTable.setColumnNames(alterPartition.getColumnNamesList().toArray(new String[alterPartition + alterTable.setPartitionColumns(alterPartition.getColumnNamesList().toArray(new String[alterPartition .getColumnNamesCount()])); alterTable.setPartitionValues(alterPartition.getPartitionValuesList().toArray(new String[alterPartition .getPartitionValuesCount()])); @@ -607,7 +607,7 @@ private static AlterTableNode convertAlterTable(PlanProto.LogicalNode protoNode) break; case DROP_PARTITION: alterPartition = alterTableProto.getAlterPartition(); - alterTable.setColumnNames(alterPartition.getColumnNamesList().toArray(new String[alterPartition + alterTable.setPartitionColumns(alterPartition.getColumnNamesList().toArray(new String[alterPartition .getColumnNamesCount()])); alterTable.setPartitionValues(alterPartition.getPartitionValuesList().toArray(new String[alterPartition .getPartitionValuesCount()])); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index e18276a4aa..8bfcde4ffb 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -573,7 +573,7 @@ public LogicalNode visitAlterTable(SerializeContext context, LogicalPlan plan, L case ADD_PARTITION: alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.ADD_PARTITION); partitionBuilder = PlanProto.AlterTableNode.AlterPartition.newBuilder(); - for (String columnName : node.getColumnNames()) { + for (String columnName : node.getPartitionColumns()) { partitionBuilder.addColumnNames(columnName); } @@ -588,7 +588,7 @@ public LogicalNode visitAlterTable(SerializeContext context, LogicalPlan plan, L case DROP_PARTITION: alterTableBuilder.setSetType(PlanProto.AlterTableNode.Type.DROP_PARTITION); partitionBuilder = PlanProto.AlterTableNode.AlterPartition.newBuilder(); - for (String columnName : node.getColumnNames()) { + for (String columnName : node.getPartitionColumns()) { partitionBuilder.addColumnNames(columnName); } From 3b57d9d4fcdd3653e33e0f00e2c593561f09590e Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Sun, 5 Jul 2015 21:26:37 +0900 Subject: [PATCH 04/25] Add purge option to drop partition statement. --- .../org/apache/tajo/algebra/AlterTable.java | 27 +++++++++++-------- .../apache/tajo/engine/parser/SQLParser.g4 | 2 +- .../tajo/engine/parser/SQLAnalyzer.java | 1 + .../tajo/engine/parser/TestSQLAnalyzer.java | 4 +++ .../default/alter_table_drop_partition_3.sql | 2 +- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java index 9440257976..260025f8ac 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java @@ -51,6 +51,9 @@ public class AlterTable extends Expr { @Expose @SerializedName("location") private String location; + @Expose @SerializedName("IsPurge") + private boolean purge; + public AlterTable(final String tableName) { super(OpType.AlterTable); this.tableName = tableName; @@ -129,18 +132,18 @@ public void setParams(Map params) { this.params = params; } + public boolean isPurge() { + return purge; + } + + public void setPurge(boolean purge) { + this.purge = purge; + } + @Override public int hashCode() { - return Objects.hashCode(tableName, - null != newTableName ? Objects.hashCode(newTableName) : newTableName, - null != columnName ? Objects.hashCode(columnName) : columnName, - null != newColumnName ? Objects.hashCode(newColumnName) : newColumnName, - null != addNewColumn ? Objects.hashCode(addNewColumn) : addNewColumn, - null != alterTableOpType ? Objects.hashCode(alterTableOpType) : alterTableOpType, - null != columns ? Objects.hashCode(columns) : columns, - null != values ? Objects.hashCode(values) : values, - null != location ? Objects.hashCode(location) : location, - null != params ? Objects.hashCode(params) : params + return Objects.hashCode(tableName, newTableName, columnName, newColumnName, addNewColumn, alterTableOpType, + columns, values, location, params, purge ); } @@ -157,7 +160,8 @@ boolean equalsTo(Expr expr) { TUtil.checkEquals(columns, another.columns) && TUtil.checkEquals(values, another.values) && TUtil.checkEquals(location, another.location) && - TUtil.checkEquals(params, another.params) + TUtil.checkEquals(params, another.params) && + TUtil.checkEquals(purge, another.purge) ; } @@ -178,6 +182,7 @@ public Object clone() throws CloneNotSupportedException { if (params != null) { alter.params = new HashMap(params); } + alter.purge = purge; return alter; } } diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 469b2a2f08..1137b30448 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -1603,7 +1603,7 @@ alter_table_statement | ALTER TABLE table_name RENAME COLUMN column_name TO column_name | ALTER TABLE table_name ADD COLUMN field_element | ALTER TABLE table_name (if_not_exists)? ADD PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (LOCATION path=Character_String_Literal)? - | ALTER TABLE table_name (if_exists)? DROP PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN + | ALTER TABLE table_name (if_exists)? DROP PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (PURGE)? | ALTER TABLE table_name SET PROPERTY property_list ; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 62bb0f9533..9f850f9405 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1847,6 +1847,7 @@ public Expr visitAlter_table_statement(SQLParser.Alter_table_statementContext ct String path = stripQuote(ctx.path.getText()); alterTable.setLocation(path); } + alterTable.setPurge(checkIfExist(ctx.PURGE())); } if (checkIfExist(ctx.property_list())) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index f698ed5978..0e37b473dd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -30,6 +30,7 @@ import java.util.Iterator; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -448,6 +449,7 @@ public void testAlterTableDropPartition1() throws IOException { assertEquals("1", value1.getValue()); LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; assertEquals("2", value2.getValue()); + assertFalse(alterTable.isPurge()); } @Test @@ -468,6 +470,7 @@ public void testAlterTableDropPartition2() throws IOException { assertEquals("01", value2.getValue()); LiteralValue value3 = (LiteralValue)alterTable.getValues()[2]; assertEquals("11", value3.getValue()); + assertFalse(alterTable.isPurge()); } @Test @@ -482,6 +485,7 @@ public void testAlterTableDropPartition3() throws IOException { assertEquals("col1", alterTable.getColumns()[0].getName()); LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; assertEquals("TAJO", value1.getValue()); + assertTrue(alterTable.isPurge()); } @Test diff --git a/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql index 1942e16889..8a1a6a5252 100644 --- a/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql +++ b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql @@ -1 +1 @@ -ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) \ No newline at end of file +ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) PURGE \ No newline at end of file From 85dbbb5ccff99b98f57f1e4966aac4425868978b Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Sun, 5 Jul 2015 23:49:48 +0900 Subject: [PATCH 05/25] Improve to manage partition's directory. --- .../org/apache/tajo/catalog/CatalogUtil.java | 54 ++++++++++++++----- .../apache/tajo/master/exec/DDLExecutor.java | 52 ++++++++++++++++++ .../tajo/engine/query/TestAlterTable.java | 13 +++-- .../org/apache/tajo/plan/LogicalPlanner.java | 1 + .../tajo/plan/logical/AlterTableNode.java | 12 ++++- .../plan/serder/LogicalNodeSerializer.java | 1 + tajo-plan/src/main/proto/Plan.proto | 1 + 7 files changed, 116 insertions(+), 18 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 7e8a63f1e2..e4b551f47c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -42,10 +42,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import static org.apache.tajo.common.TajoDataTypes.Type; @@ -794,13 +791,50 @@ public static AlterTableDesc setProperty(String tableName, KeyValueSet params, A return alterTableDesc; } + /** + * Add partition or Drop partition + * + * @param tableName table name + * @param columns partition column names + * @param values partition values + * @param location partition location + * @param alterTableType ADD_PARTITION or DROP_PARTITION + * @return + */ public static AlterTableDesc addOrDropPartition(String tableName, String[] columns, String[] values, String location, AlterTableType alterTableType) { final AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(tableName); PartitionDesc partitionDesc = new PartitionDesc(); + Pair, String> pair = getPartitionKeyNamePair(columns, values); + + partitionDesc.setPartitionKeys(pair.getFirst()); + partitionDesc.setPartitionName(pair.getSecond()); + + if (alterTableType.equals(AlterTableType.ADD_PARTITION) && location != null) { + partitionDesc.setPath(location); + } + + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(alterTableType); + return alterTableDesc; + } + /** + * Get partition key/value list and partition name + * + * ex) partition key/value list : + * - col1, 2015-07-01 + * - col2, tajo + * partition name : col1=2015-07-01/col2=tajo + * + * @param columns partition column names + * @param values partition values + * @return partition key/value list and partition name + */ + public static Pair, String> getPartitionKeyNamePair(String[] columns, String[] values) { + Pair, String> pair = null; List partitionKeyList = TUtil.newList(); StringBuilder sb = new StringBuilder(); @@ -816,16 +850,8 @@ public static AlterTableDesc addOrDropPartition(String tableName, String[] colum partitionKeyList.add(partitionKey); } - partitionDesc.setPartitionKeys(partitionKeyList); - partitionDesc.setPartitionName(sb.toString()); - - if (alterTableType.equals(AlterTableType.ADD_PARTITION) && location != null) { - partitionDesc.setPath(location); - } - - alterTableDesc.setPartitionDesc(partitionDesc); - alterTableDesc.setAlterTableType(alterTableType); - return alterTableDesc; + pair = new Pair, String>(partitionKeyList, sb.toString()); + return pair; } /* It is the relationship graph of type conversions. */ diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index f81bad22b9..6695174014 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -29,7 +29,10 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.exception.NoSuchColumnException; +import org.apache.tajo.catalog.partition.PartitionDesc; +import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.query.QueryContext; @@ -40,6 +43,7 @@ import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.util.Pair; import java.io.IOException; import java.net.URI; @@ -449,19 +453,67 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer break; case ADD_PARTITION: existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); + + Path partitionPath = null; + + if (alterTable.getLocation() != null) { + partitionPath = new Path(alterTable.getLocation()); + } else { + // If location is not specified, the partition's location will be set using the table location. + desc = catalog.getTableDesc(databaseName, simpleTableName); + Pair, String> pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), + alterTable.getPartitionValues()); + partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); + alterTable.setLocation(partitionPath.toString()); + } + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION)); + + FileSystem fs = partitionPath.getFileSystem(context.getConf()); + // If the partition's path doesn't exist, this would make the directory by force. + if (!fs.exists(partitionPath)) { + fs.mkdirs(partitionPath); + } break; case DROP_PARTITION: existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); + + desc = catalog.getTableDesc(databaseName, simpleTableName); + + Pair, String> pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), + alterTable.getPartitionValues()); + + CatalogProtos.PartitionDescProto partitionDescProto = catalog.getPartition(databaseName, simpleTableName, + pair.getSecond()); + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.DROP_PARTITION)); + + // When dropping partition on an managed table, the data will be delete from file system. + if (!desc.isExternal()) { + deletePartitionPath(partitionDescProto); + } else { + // When dropping partition on an external table, the data in the table will NOT be deleted from the file + // system. But if PURGE is specified, the partition data will be deleted. + if (alterTable.isPurge()) { + deletePartitionPath(partitionDescProto); + } + } break; default: //TODO } } + private void deletePartitionPath(CatalogProtos.PartitionDescProto partitionDescProto) throws IOException { + Path partitionPath = new Path(partitionDescProto.getPath()); + FileSystem fs = partitionPath.getFileSystem(context.getConf()); + if (fs.exists(partitionPath)) { + fs.delete(partitionPath, true); + } + } + private boolean existPartitionColumnNames(String tableName, String[] columnNames) { for(String columnName : columnNames) { if (!existPartitionColumnName(tableName, columnName)) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index c692c24b3c..8cdaf80d8a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -18,6 +18,8 @@ package org.apache.tajo.engine.query; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.CatalogUtil; @@ -30,9 +32,7 @@ import java.util.List; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; @Category(IntegrationTest.class) public class TestAlterTable extends QueryTestCaseBase { @@ -97,10 +97,17 @@ public final void testAlterTableAddPartition() throws Exception { assertEquals(partitions.get(0).getPartitionKeysList().get(1).getColumnName(), "col4"); assertEquals(partitions.get(0).getPartitionKeysList().get(1).getPartitionValue(), "2"); + assertNotNull(partitions.get(0).getPath()); + Path partitionPath = new Path(partitions.get(0).getPath()); + FileSystem fs = partitionPath.getFileSystem(conf); + assertTrue(fs.exists(partitionPath)); + assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); + executeDDL("alter_table_drop_partition1.sql", null); partitions = catalog.getPartitions("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 0); + assertFalse(fs.exists(partitionPath)); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index bdc8d81486..f8b90ca618 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -2053,6 +2053,7 @@ public LogicalNode visitAlterTable(PlanContext context, Stack stack, Alter alterTableNode.setLocation(alterTable.getLocation()); } + alterTableNode.setPurge(alterTable.isPurge()); alterTableNode.setAlterTableOpType(alterTable.getAlterTableOpType()); return alterTableNode; } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java index 8459aeb436..ecb173a584 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java @@ -49,6 +49,8 @@ public class AlterTableNode extends LogicalNode { private String[] partitionValues; @Expose private String location; + @Expose + private boolean isPurge; public AlterTableNode(int pid) { super(pid, NodeType.ALTER_TABLE); @@ -148,6 +150,14 @@ public void setLocation(String location) { this.location = location; } + public boolean isPurge() { + return isPurge; + } + + public void setPurge(boolean isPurge) { + this.isPurge = isPurge; + } + @Override public PlanString getPlanString() { return new PlanString(this); @@ -156,7 +166,7 @@ public PlanString getPlanString() { @Override public int hashCode() { return Objects.hashCode(tableName, addNewColumn, alterTableOpType, columnName, newColumnName, newTableName, - tableName, properties, partitionColumns, partitionValues, location); + tableName, properties, partitionColumns, partitionValues, location, isPurge); } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java index 8bfcde4ffb..76370a9e32 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeSerializer.java @@ -595,6 +595,7 @@ public LogicalNode visitAlterTable(SerializeContext context, LogicalPlan plan, L for (String partitionValue : node.getPartitionValues()) { partitionBuilder.addPartitionValues(partitionValue); } + partitionBuilder.setPurge(node.isPurge()); alterTableBuilder.setAlterPartition(partitionBuilder); break; default: diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 25f51d699f..7855acd780 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -301,6 +301,7 @@ message AlterTableNode { repeated string columnNames = 1; repeated string partitionValues = 21; optional string location = 3; + optional bool purge = 4; } required string tableName = 1; From 279a01a2121e7e6f99c9a0575d760069cbc9b157 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 6 Jul 2015 00:44:35 +0900 Subject: [PATCH 06/25] Fix TajoCli test error and add a document for alter table statement. --- .../testAlterTableDropPartition.result | 2 +- tajo-docs/src/main/sphinx/sql_language.rst | 3 +- .../main/sphinx/sql_language/alter_table.rst | 100 ++++++++++++++++++ 3 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 tajo-docs/src/main/sphinx/sql_language/alter_table.rst diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result index bcaba86266..adf66bfa38 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result @@ -1,2 +1,2 @@ OK -OK \ No newline at end of file +ERROR: java.lang.NullPointerException \ No newline at end of file diff --git a/tajo-docs/src/main/sphinx/sql_language.rst b/tajo-docs/src/main/sphinx/sql_language.rst index 1b405e6b8b..b56576597b 100644 --- a/tajo-docs/src/main/sphinx/sql_language.rst +++ b/tajo-docs/src/main/sphinx/sql_language.rst @@ -10,4 +10,5 @@ SQL Language sql_language/insert sql_language/queries sql_language/sql_expression - sql_language/predicates \ No newline at end of file + sql_language/predicates + sql_language/alter_table diff --git a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst new file mode 100644 index 0000000000..e29fa99711 --- /dev/null +++ b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst @@ -0,0 +1,100 @@ +************************ +ALTER TABLE +************************ + +======================== +RENAME TABLE +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE RENAME TO + + For example: + ALTER TABLE table1 RENAME TO table2; + +This statement lets you change the name of a table to a different name. + +======================== +RENAME COLUMN +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE RENAME COLUMN TO + + For example: + ALTER TABLE table1 RENAME COLUMN id TO id2; + +This statement will allow users to change a column's name. + +======================== +ADD COLUMN +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE ADD COLUMN + + For example: + ALTER TABLE table1 ADD COLUMN id text; + +This statement lets you add new columns to the end of the existing column. + +======================== +SET PROPERTY +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE SET PROPERTY ( = , ...) + + For example: + ALTER TABLE table1 SET PROPERTY 'timezone' = 'GMT-7' + ALTER TABLE table1 SET PROPERTY 'text.delimiter' = '&' + ALTER TABLE table1 SET PROPERTY 'compression.type'='RECORD','compression.codec'='org.apache.hadoop.io.compress.SnappyCodec' + + +This statement will allow users to change a table's properties. + +======================== +ADD PARTITION +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE [IF NOT EXISTS] ADD PARTITION ( = , ...) [LOCATION = ] + + For example: + ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) + ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx.com/warehouse/table1/col1=1/col2=2' + +You can use ALTER TABLE ADD PARTITION to add partitions to a table. The location must be a directory inside of which data files reside. If the location doesn't exist on the file system, Tajo will make the location by force. +ADD PARTITION changes the table metadata, but does not load data. If the data does not exist in the partition's location, queries will not return any results. + +======================== + DROP PARTITION +======================== + +*Synopsis* + +.. code-block:: sql + + ALTER TABLE [IF NOT EXISTS] DROP PARTITION ( = , ...) [PURGE] + + For example: + ALTER TABLE table1 DROP PARTITION (col1 = 1 , col2 = 2) + ALTER TABLE table1 DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) + ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) PURGE + +You can use ALTER TABLE DROP PARTITION to drop a partition for a table. This removes the data for a managed table and this doesn't remove the data for an external table. But if PURGE is specified for an external table, the partition data will be removed. For the reference, the metadata is completely lost in all cases. \ No newline at end of file From 9bb8c506cca8feb3ffad840babbc7a3b80e3084b Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Mon, 6 Jul 2015 00:48:46 +0900 Subject: [PATCH 07/25] Update a document for alter table statement. --- tajo-docs/src/main/sphinx/sql_language/alter_table.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst index e29fa99711..3f89b775d4 100644 --- a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst +++ b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst @@ -1,5 +1,5 @@ ************************ -ALTER TABLE +ALTER TABLE Statement ************************ ======================== @@ -79,8 +79,7 @@ ADD PARTITION ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx.com/warehouse/table1/col1=1/col2=2' -You can use ALTER TABLE ADD PARTITION to add partitions to a table. The location must be a directory inside of which data files reside. If the location doesn't exist on the file system, Tajo will make the location by force. -ADD PARTITION changes the table metadata, but does not load data. If the data does not exist in the partition's location, queries will not return any results. +You can use ``ALTER TABLE ADD PARTITION`` to add partitions to a table. The location must be a directory inside of which data files reside. If the location doesn't exist on the file system, Tajo will make the location by force. ``ADD PARTITION`` changes the table metadata, but does not load data. If the data does not exist in the partition's location, queries will not return any results. ======================== DROP PARTITION @@ -97,4 +96,4 @@ ADD PARTITION changes the table metadata, but does not load data. If the data do ALTER TABLE table1 DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) PURGE -You can use ALTER TABLE DROP PARTITION to drop a partition for a table. This removes the data for a managed table and this doesn't remove the data for an external table. But if PURGE is specified for an external table, the partition data will be removed. For the reference, the metadata is completely lost in all cases. \ No newline at end of file +You can use ``ALTER TABLE DROP PARTITION`` to drop a partition for a table. This removes the data for a managed table and this doesn't remove the data for an external table. But if ``PURGE`` is specified for an external table, the partition data will be removed. For the reference, the metadata is completely lost in all cases. \ No newline at end of file From 7f5f8d73aa3d33c61e677819c183f02a844f4522 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 7 Jul 2015 09:31:10 +0900 Subject: [PATCH 08/25] Update an unit test case. --- .../java/org/apache/tajo/cli/tsql/TestTajoCli.java | 12 +----------- ....result => testAlterTableAddDropPartition.result} | 1 + .../TestTajoCli/testAlterTableDropPartition.result | 2 -- 3 files changed, 2 insertions(+), 13 deletions(-) rename tajo-core/src/test/resources/results/TestTajoCli/{testAlterTableAddPartition.result => testAlterTableAddDropPartition.result} (62%) delete mode 100644 tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index 26e25a4d6a..d2771fb305 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -431,21 +431,11 @@ public void testNonForwardQueryPause() throws Exception { } @Test - public void testAlterTableAddPartition() throws Exception { + public void testAlterTableAddDropPartition() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition"); tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); tajoCli.executeScript("alter table " + tableName + " add partition (key = 0.1)"); - - String consoleResult = new String(out.toByteArray()); - assertOutputResult(consoleResult); - } - - @Test - public void testAlterTableDropPartition() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testAlterTableDropPartition"); - - tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)"); String consoleResult = new String(out.toByteArray()); diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result similarity index 62% rename from tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result rename to tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result index bcaba86266..7f3546976f 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result @@ -1,2 +1,3 @@ OK +OK OK \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result deleted file mode 100644 index adf66bfa38..0000000000 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result +++ /dev/null @@ -1,2 +0,0 @@ -OK -ERROR: java.lang.NullPointerException \ No newline at end of file From ff66f5725aa0572ab5ca8a5f5e713a93a4cd6fd6 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 7 Jul 2015 09:35:04 +0900 Subject: [PATCH 09/25] Update a document for alter table statement. --- tajo-docs/src/main/sphinx/sql_language/alter_table.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst index 3f89b775d4..9206625783 100644 --- a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst +++ b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst @@ -63,7 +63,7 @@ SET PROPERTY ALTER TABLE table1 SET PROPERTY 'compression.type'='RECORD','compression.codec'='org.apache.hadoop.io.compress.SnappyCodec' -This statement will allow users to change a table's properties. +This statement will allow users to change a table property. ======================== ADD PARTITION @@ -96,4 +96,4 @@ You can use ``ALTER TABLE ADD PARTITION`` to add partitions to a table. The loca ALTER TABLE table1 DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) PURGE -You can use ``ALTER TABLE DROP PARTITION`` to drop a partition for a table. This removes the data for a managed table and this doesn't remove the data for an external table. But if ``PURGE`` is specified for an external table, the partition data will be removed. For the reference, the metadata is completely lost in all cases. \ No newline at end of file +You can use ``ALTER TABLE DROP PARTITION`` to drop a partition for a table. This removes the data for a managed table and this doesn't remove the data for an external table. But if ``PURGE`` is specified for an external table, the partition data will be removed. For your information, the metadata is completely lost in all cases. \ No newline at end of file From a88772fd0a2b6c3ef8815974b3d217d3ad4c26be Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 7 Jul 2015 11:09:37 +0900 Subject: [PATCH 10/25] Update exception message and add unit test cases. --- .../catalog/exception/NoPartitionedTableException.java | 2 +- .../tajo/catalog/exception/NoSuchPartitionException.java | 4 ++++ .../catalog/exception/NoSuchPartitionKeyException.java | 7 ++++++- .../java/org/apache/tajo/master/exec/DDLExecutor.java | 8 +++++--- .../test/java/org/apache/tajo/cli/tsql/TestTajoCli.java | 2 ++ .../TestTajoCli/testAlterTableAddDropPartition.result | 4 +++- 6 files changed, 21 insertions(+), 6 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java index faa8bc407a..eded78922b 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoPartitionedTableException.java @@ -23,6 +23,6 @@ public class NoPartitionedTableException extends Exception { public NoPartitionedTableException() {} public NoPartitionedTableException(String databaseName, String relName) { - super(String.format("ERROR: \"%s.%s\" is not a partitioned table", databaseName, relName)); + super(String.format("ERROR: table \"%s.%s\" is not a partitioned table", databaseName, relName)); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java index 45c92990bf..78f9e2d110 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java @@ -32,6 +32,10 @@ public NoSuchPartitionException(String message) { super(message); } + public NoSuchPartitionException(String tableName, String partitionName) { + super(String.format("ERROR: \"%s\" does not exist in \"%s\".", partitionName, tableName)); + } + public NoSuchPartitionException(String databaseName, String tableName, String partitionName) { super(String.format("ERROR: \"%s\" does not exist in \"%s.%s\".", partitionName, databaseName, tableName)); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java index bc56580655..add4bbb127 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java @@ -32,8 +32,13 @@ public NoSuchPartitionKeyException(String message) { super(message); } + public NoSuchPartitionKeyException(String tableName, String partitionKey) { + super(String.format("ERROR: partition column \"%s\" does not exist in \"%s\".", partitionKey, tableName)); + } + public NoSuchPartitionKeyException(String databaseName, String tableName, String partitionKey) { - super(String.format("ERROR: \"%s\" does not exist in \"%s.%s\".", partitionKey, databaseName, tableName)); + super(String.format("ERROR: partition column \"%s\" does not exist in \"%s.%s\".", partitionKey, + databaseName, tableName)); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 6695174014..c504844b08 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -453,7 +453,6 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer break; case ADD_PARTITION: existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); - Path partitionPath = null; if (alterTable.getLocation() != null) { @@ -478,7 +477,6 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer break; case DROP_PARTITION: existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); - desc = catalog.getTableDesc(databaseName, simpleTableName); Pair, String> pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), @@ -487,6 +485,10 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer CatalogProtos.PartitionDescProto partitionDescProto = catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); + if (partitionDescProto == null) { + throw new NoSuchPartitionException(tableName, pair.getSecond()); + } + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.DROP_PARTITION)); @@ -517,7 +519,7 @@ private void deletePartitionPath(CatalogProtos.PartitionDescProto partitionDescP private boolean existPartitionColumnNames(String tableName, String[] columnNames) { for(String columnName : columnNames) { if (!existPartitionColumnName(tableName, columnName)) { - throw new NoSuchPartitionKeyException(columnName); + throw new NoSuchPartitionKeyException(tableName, columnName); } } return true; diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index d2771fb305..9cf1535b0d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -435,8 +435,10 @@ public void testAlterTableAddDropPartition() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition"); tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); + tajoCli.executeScript("alter table " + tableName + " add partition (key2 = 0.1)"); tajoCli.executeScript("alter table " + tableName + " add partition (key = 0.1)"); tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)"); + tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)"); String consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result index 7f3546976f..98f0a0c090 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result @@ -1,3 +1,5 @@ OK +ERROR: partition column "key2" does not exist in "default.testaltertableaddpartition". OK -OK \ No newline at end of file +OK +ERROR: "key=0.1" does not exist in "testaltertableaddpartition". \ No newline at end of file From 6061843229e8c79bbd1fbd13fe0366a3e7837ed9 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 7 Jul 2015 17:35:06 +0900 Subject: [PATCH 11/25] Update exception message and add more unit test cases. --- .../AlreadyExistsPartitionException.java | 6 ++- .../apache/tajo/master/exec/DDLExecutor.java | 41 +++++++++++-------- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 20 ++++++++- .../testAlterTableAddDropPartition.result | 6 ++- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java index ab6144f26a..a7f0d87934 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsPartitionException.java @@ -26,8 +26,12 @@ public AlreadyExistsPartitionException(String message) { super(message); } + public AlreadyExistsPartitionException(String tableName, String partitionName) { + super(String.format("ERROR: partition \"%s already exist in \"%s\"", partitionName, tableName)); + } + public AlreadyExistsPartitionException(String databaseName, String tableName, String partitionName) { - super(String.format("ERROR: \"%s already exist in \"%s.%s\"", partitionName, databaseName, tableName)); + super(String.format("ERROR: partition \"%s already exist in \"%s.%s\"", partitionName, databaseName, tableName)); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index c504844b08..7fb41712f0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -24,12 +24,11 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.algebra.AlterTableOpType; import org.apache.tajo.algebra.AlterTablespaceSetType; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.*; -import org.apache.tajo.catalog.exception.NoSuchColumnException; -import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -405,6 +404,25 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer throw new NoSuchTableException(qualifiedName); } + Path partitionPath = null; + TableDesc desc = null; + Pair, String> pair = null; + CatalogProtos.PartitionDescProto partitionDescProto = null; + + if (alterTable.getAlterTableOpType() == AlterTableOpType.RENAME_TABLE + || alterTable.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION + || alterTable.getAlterTableOpType() == AlterTableOpType.DROP_PARTITION) { + desc = catalog.getTableDesc(databaseName, simpleTableName); + } + + // When adding a partition or dropping a partition, check existing partition column information. + if (alterTable.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION + || alterTable.getAlterTableOpType() == AlterTableOpType.DROP_PARTITION) { + pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), alterTable.getPartitionValues()); + partitionDescProto = catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); + existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); + } + switch (alterTable.getAlterTableOpType()) { case RENAME_TABLE: if (!catalog.existsTable(databaseName, simpleTableName)) { @@ -414,8 +432,6 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer throw new AlreadyExistsTableException(alterTable.getNewTableName()); } - TableDesc desc = catalog.getTableDesc(databaseName, simpleTableName); - if (!desc.isExternal()) { // if the table is the managed table Path oldPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR), databaseName, simpleTableName); @@ -452,16 +468,14 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer catalog.alterTable(CatalogUtil.setProperty(qualifiedName, alterTable.getProperties(), AlterTableType.SET_PROPERTY)); break; case ADD_PARTITION: - existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); - Path partitionPath = null; + if (partitionDescProto != null) { + throw new AlreadyExistsPartitionException(tableName, pair.getSecond()); + } if (alterTable.getLocation() != null) { partitionPath = new Path(alterTable.getLocation()); } else { // If location is not specified, the partition's location will be set using the table location. - desc = catalog.getTableDesc(databaseName, simpleTableName); - Pair, String> pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), - alterTable.getPartitionValues()); partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); alterTable.setLocation(partitionPath.toString()); } @@ -476,15 +490,6 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer } break; case DROP_PARTITION: - existPartitionColumnNames(qualifiedName, alterTable.getPartitionColumns()); - desc = catalog.getTableDesc(databaseName, simpleTableName); - - Pair, String> pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), - alterTable.getPartitionValues()); - - CatalogProtos.PartitionDescProto partitionDescProto = catalog.getPartition(databaseName, simpleTableName, - pair.getSecond()); - if (partitionDescProto == null) { throw new NoSuchPartitionException(tableName, pair.getSecond()); } diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index 9cf1535b0d..5005670045 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -30,6 +30,7 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; +import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TablespaceManager; @@ -85,7 +86,7 @@ public void tearDown() throws IOException { } private static void setVar(TajoCli cli, ConfigKey key, String val) throws Exception { - cli.executeMetaCommand("\\set " + key.keyname() +" " + val); + cli.executeMetaCommand("\\set " + key.keyname() + " " + val); } private static void assertSessionVar(TajoCli cli, String key, String expectedVal) { @@ -440,6 +441,23 @@ public void testAlterTableAddDropPartition() throws Exception { tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)"); tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)"); + tajoCli.executeScript("drop table " + tableName); + tajoCli.executeScript("create table " + tableName + + " (col1 int4, col2 int4) partition by column(col3 float8, col4 int4)"); + + TajoClient client = testBase.getTestingCluster().newTajoClient(); + TableDesc tableDesc = client.getTableDesc(tableName); + + String partitionLocation = tableDesc.getUri().toString() + "/col5=0.1/col6=10"; + tajoCli.executeScript("alter table " + tableName + " add partition (col3 = 0.1, col4 = 10)" + + " location '" + partitionLocation + "'"); + + Path partitionPath = new Path(partitionLocation); + FileSystem fs = testBase.getTestingCluster().getDefaultFileSystem(); + assertTrue(fs.exists(partitionPath)); + + tajoCli.executeScript("alter table " + tableName + " drop partition (col3 = 0.1, col4 = 10)"); + String consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); } diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result index 98f0a0c090..d5e0f65bca 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result @@ -2,4 +2,8 @@ OK ERROR: partition column "key2" does not exist in "default.testaltertableaddpartition". OK OK -ERROR: "key=0.1" does not exist in "testaltertableaddpartition". \ No newline at end of file +ERROR: "key=0.1" does not exist in "testaltertableaddpartition". +OK +OK +OK +OK \ No newline at end of file From 0c68872460dd5058a2fddb5022fa5d641c49e2c1 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 7 Jul 2015 19:12:27 +0900 Subject: [PATCH 12/25] Trigger for Travis CI build. --- .../src/main/java/org/apache/tajo/master/exec/DDLExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 7fb41712f0..311efea852 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -483,8 +483,8 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION)); - FileSystem fs = partitionPath.getFileSystem(context.getConf()); // If the partition's path doesn't exist, this would make the directory by force. + FileSystem fs = partitionPath.getFileSystem(context.getConf()); if (!fs.exists(partitionPath)) { fs.mkdirs(partitionPath); } From 2a4257bbba05c230d3459b256983cc31017a679b Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 8 Jul 2015 09:44:05 +0900 Subject: [PATCH 13/25] Update log level for TajoSystemMetrics and disable unit test cases for unsupported partition type. --- .../java/org/apache/tajo/util/metrics/TajoSystemMetrics.java | 4 ++-- .../src/test/java/org/apache/tajo/client/TestTajoClient.java | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java index 4192ca0a40..ac26beb09e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java @@ -128,8 +128,8 @@ private void setMetricsReporter(String groupName) { Map reporters = new HashMap(); List reporterNames = metricsProps.getList(groupName + ".reporters"); - if(reporterNames.isEmpty()) { - LOG.warn("No property " + groupName + ".reporters in " + metricsPropertyFileName); + if(reporterNames.isEmpty() && LOG.isDebugEnabled()) { + LOG.debug("No property " + groupName + ".reporters in " + metricsPropertyFileName); return; } diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index 73b97fa37f..fe8fb58297 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -407,6 +407,7 @@ public final void testGetTableDesc() throws IOException, ServiceException, SQLEx assertTrue(desc.getStats().getNumBytes() > 0); } + /* //@Test public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery() throws IOException, ServiceException, SQLException { @@ -529,6 +530,7 @@ public final void testCreateAndDropTablePartitionedRangeByExecuteQuery() throws assertFalse(client.existTable(tableName)); assertFalse(hdfs.exists(tablePath)); } + */ @Test public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOException, From e99202e353ef5c0407eb53c413c79100b5c76327 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 8 Jul 2015 09:46:45 +0900 Subject: [PATCH 14/25] Add trivial comment. --- .../src/test/java/org/apache/tajo/client/TestTajoClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index fe8fb58297..3d8857fa44 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -407,6 +407,7 @@ public final void testGetTableDesc() throws IOException, ServiceException, SQLEx assertTrue(desc.getStats().getNumBytes() > 0); } + // TODO: If tajo support hash, list, rage partition, following codes would be used. /* //@Test public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery() throws IOException, From 5450dfef65bb29be02dee1e8a70ae31aee946880 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 8 Jul 2015 09:53:41 +0900 Subject: [PATCH 15/25] Add batch mode for the Travis CI build. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 61d56fbdfa..4cb8055e9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,4 +37,4 @@ before_install: ulimit -t 514029 install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 + mvn clean install -B -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 From 29bca3240a6bdf7ae2cbeb52c4c065feab364d4f Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 8 Jul 2015 10:42:15 +0900 Subject: [PATCH 16/25] Rollback removed test codes. --- .../apache/tajo/client/TestTajoClient.java | 65 +++++++++---------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index 3d8857fa44..6ae52bc04b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -227,8 +227,8 @@ public final void testUpdateQuery() throws IOException, ServiceException { assertFalse(client.existTable(tableName)); String sql = - "create external table " + tableName + " (deptname text, score integer) " - + "using csv location '" + tablePath + "'"; + "create external table " + tableName + " (deptname text, score integer) " + + "using csv location '" + tablePath + "'"; client.updateQuery(sql); assertTrue(client.existTable(tableName)); client.dropTable(tableName); @@ -237,7 +237,7 @@ public final void testUpdateQuery() throws IOException, ServiceException { @Test public final void testCreateAndDropExternalTable() - throws IOException, ServiceException, SQLException { + throws IOException, ServiceException, SQLException { final String tableName = "testCreateAndDropExternalTable"; Path tablePath = writeTmpTable(tableName); LOG.error("Full path:" + tablePath.toUri().getRawPath()); @@ -281,7 +281,7 @@ public final void testCreateAndDropExternalTableByExecuteQuery() throws IOExcept assertFalse(client.existTable(tableName)); String sql = "create external table " + tableName + " (deptname text, score int4) " + "using csv location '" - + tablePath + "'"; + + tablePath + "'"; client.executeQueryAndGetResult(sql); assertTrue(client.existTable(tableName)); @@ -301,7 +301,7 @@ public final void testCreateAndPurgeExternalTableByExecuteQuery() throws IOExcep assertFalse(client.existTable(tableName)); String sql = "create external table " + tableName + " (deptname text, score int4) " + "using csv location '" - + tablePath + "'"; + + tablePath + "'"; client.executeQueryAndGetResult(sql); assertTrue(client.existTable(tableName)); @@ -361,8 +361,8 @@ public final void testDDLByExecuteQuery() throws IOException, ServiceException { assertFalse(client.existTable(tableName)); String sql = - "create external table " + tableName + " (deptname text, score int4) " - + "using csv location '" + tablePath + "'"; + "create external table " + tableName + " (deptname text, score int4) " + + "using csv location '" + tablePath + "'"; client.executeQueryAndGetResult(sql); assertTrue(client.existTable(tableName)); } @@ -407,11 +407,9 @@ public final void testGetTableDesc() throws IOException, ServiceException, SQLEx assertTrue(desc.getStats().getNumBytes() > 0); } - // TODO: If tajo support hash, list, rage partition, following codes would be used. - /* //@Test public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndDropTablePartitionedHash1ByExecuteQuery"; @@ -435,7 +433,7 @@ public final void testCreateAndDropTablePartitionedHash1ByExecuteQuery() throws //@Test public final void testCreateAndPurgeTablePartitionedHash1ByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndPurgeTablePartitionedHash1ByExecuteQuery"; @@ -459,7 +457,7 @@ public final void testCreateAndPurgeTablePartitionedHash1ByExecuteQuery() throws //@Test public final void testCreateAndDropTablePartitionedHash2ByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndDropTablePartitionedHash2ByExecuteQuery"; @@ -483,7 +481,7 @@ public final void testCreateAndDropTablePartitionedHash2ByExecuteQuery() throws //@Test public final void testCreateAndDropTablePartitionedListByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndDropTablePartitionedListByExecuteQuery"; @@ -508,7 +506,7 @@ public final void testCreateAndDropTablePartitionedListByExecuteQuery() throws I //@Test public final void testCreateAndDropTablePartitionedRangeByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testCreateAndDropTablePartitionedRangeByExecuteQuery"; @@ -531,11 +529,10 @@ public final void testCreateAndDropTablePartitionedRangeByExecuteQuery() throws assertFalse(client.existTable(tableName)); assertFalse(hdfs.exists(tablePath)); } - */ @Test public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = "testFailCreateTablePartitionedOtherExceptColumn"; @@ -548,7 +545,7 @@ public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOExc rangeSql += "PARTITION sub_part2 VALUES LESS THAN (MAXVALUE) )"; assertFalse(client.updateQuery(rangeSql)); - + String listSql = "create table " + tableName + " (deptname text, score int4)"; listSql += "PARTITION BY LIST (deptname)"; listSql += "( PARTITION sub_part1 VALUES('r&d', 'design'),"; @@ -565,7 +562,7 @@ public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOExc @Test public final void testCreateAndDropTablePartitionedColumnByExecuteQuery() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { TajoConf conf = cluster.getConfiguration(); final String tableName = CatalogUtil.normalizeIdentifier("testCreateAndDropTablePartitionedColumnByExecuteQuery"); @@ -588,7 +585,7 @@ public final void testCreateAndDropTablePartitionedColumnByExecuteQuery() throws @Test public final void testGetFunctions() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { Collection catalogFunctions = cluster.getMaster().getCatalog().getFunctions(); String functionName = "sum"; int numFunctions = 0; @@ -610,7 +607,7 @@ public final void testGetFunctions() throws IOException, @Test public final void testGetFinishedQueryList() throws IOException, - ServiceException, SQLException { + ServiceException, SQLException { final String tableName = CatalogUtil.normalizeIdentifier("testGetFinishedQueryList"); String sql = "create table " + tableName + " (deptname text, score int4)"; @@ -681,15 +678,15 @@ public final void testGetQueryStatusAndResultAfterFinish() throws Exception { @Test public void testNullCharSessionInCTAS() throws Exception { String sql = - "create table nullcharsession as select\n" + - " c_custkey,\n" + - " orders.o_orderkey,\n" + - " orders.o_orderstatus \n" + - "from\n" + - " orders full outer join customer on c_custkey = o_orderkey\n" + - "order by\n" + - " c_custkey,\n" + - " orders.o_orderkey;\n"; + "create table nullcharsession as select\n" + + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; Map variables = new HashMap(); variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T"); @@ -723,10 +720,10 @@ public void assertNullCharSessionVar(TableDesc resultDesc) throws Exception { // text type field's value is replaced with \T String expected = "1|1|O\n" + - "2|2|O\n" + - "3|3|F\n" + - "4||\\T\n" + - "5||\\T\n"; + "2|2|O\n" + + "3|3|F\n" + + "4||\\T\n" + + "5||\\T\n"; String resultDatas = new String(buf, 0, readBytes); @@ -760,7 +757,7 @@ public void testGetQueryInfoAndHistory() throws Exception { assertEquals(2, queryHistory.getStageHistoriesCount()); List taskHistories = - new ArrayList(queryHistory.getStageHistoriesList()); + new ArrayList(queryHistory.getStageHistoriesList()); Collections.sort(taskHistories, new Comparator() { @Override public int compare(ClientProtos.StageHistoryProto o1, StageHistoryProto o2) { From 4d5ee1f01e9664f8046213a62c57e2db64ca5cd6 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 8 Jul 2015 11:57:36 +0900 Subject: [PATCH 17/25] Remove codes for Travis CI build. --- .travis.yml | 2 +- .../java/org/apache/tajo/util/metrics/TajoSystemMetrics.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4cb8055e9e..61d56fbdfa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,4 +37,4 @@ before_install: ulimit -t 514029 install: ./dev-support/travis-install-dependencies.sh script: - mvn clean install -B -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 + mvn clean install -Pparallel-test -DLOG_LEVEL=WARN -Dmaven.fork.count=2 diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java index ac26beb09e..4192ca0a40 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java @@ -128,8 +128,8 @@ private void setMetricsReporter(String groupName) { Map reporters = new HashMap(); List reporterNames = metricsProps.getList(groupName + ".reporters"); - if(reporterNames.isEmpty() && LOG.isDebugEnabled()) { - LOG.debug("No property " + groupName + ".reporters in " + metricsPropertyFileName); + if(reporterNames.isEmpty()) { + LOG.warn("No property " + groupName + ".reporters in " + metricsPropertyFileName); return; } From e2511183942b2fed1c487c9352093d5f9b05df5a Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 10 Jul 2015 00:41:16 +0900 Subject: [PATCH 18/25] Just trigger for the Travis CI build. --- .../src/main/java/org/apache/tajo/master/exec/DDLExecutor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 311efea852..f49ea274f4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -384,7 +384,6 @@ public void truncateTable(final QueryContext queryContext, final TruncateTableNo */ public void alterTable(TajoMaster.MasterContext context, final QueryContext queryContext, final AlterTableNode alterTable) throws IOException { - final CatalogService catalog = context.getCatalog(); final String tableName = alterTable.getTableName(); From 9e8d77de5a7e96c69dc701fa148d7c3bb93c62d1 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 14 Jul 2015 22:59:29 +0900 Subject: [PATCH 19/25] Update comment for drop partition --- tajo-docs/src/main/sphinx/sql_language/alter_table.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst index 9206625783..947255cd2f 100644 --- a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst +++ b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst @@ -96,4 +96,5 @@ You can use ``ALTER TABLE ADD PARTITION`` to add partitions to a table. The loca ALTER TABLE table1 DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) PURGE -You can use ``ALTER TABLE DROP PARTITION`` to drop a partition for a table. This removes the data for a managed table and this doesn't remove the data for an external table. But if ``PURGE`` is specified for an external table, the partition data will be removed. For your information, the metadata is completely lost in all cases. \ No newline at end of file +You can use ``ALTER TABLE DROP PARTITION`` to drop a partition for a table. This removes the data for a managed table + and this doesn't remove the data for an external table. But if ``PURGE`` is specified for an external table, the partition data will be removed. The metadata is completely lost in all cases. \ No newline at end of file From 112c651c4f90a9aa29d3ffcd7897d1e661b1d0ab Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 14 Jul 2015 23:02:59 +0900 Subject: [PATCH 20/25] Remove unused codes. --- .../exception/NoSuchPartitionException.java | 6 ------ .../exception/NoSuchPartitionKeyException.java | 16 ---------------- 2 files changed, 22 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java index 78f9e2d110..d0b266a87e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java @@ -18,12 +18,6 @@ package org.apache.tajo.catalog.exception; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.function.FunctionUtil; -import org.codehaus.jackson.schema.JsonSerializableSchema; - -import java.util.Collection; - public class NoSuchPartitionException extends RuntimeException { private static final long serialVersionUID = 277182608283894938L; diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java index add4bbb127..c6bcd23793 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java @@ -18,27 +18,11 @@ package org.apache.tajo.catalog.exception; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.function.FunctionUtil; -import org.codehaus.jackson.schema.JsonSerializableSchema; - -import java.util.Collection; - public class NoSuchPartitionKeyException extends RuntimeException { private static final long serialVersionUID = 277182608283894939L; - public NoSuchPartitionKeyException(String message) { - super(message); - } - public NoSuchPartitionKeyException(String tableName, String partitionKey) { super(String.format("ERROR: partition column \"%s\" does not exist in \"%s\".", partitionKey, tableName)); } - - public NoSuchPartitionKeyException(String databaseName, String tableName, String partitionKey) { - super(String.format("ERROR: partition column \"%s\" does not exist in \"%s.%s\".", partitionKey, - databaseName, tableName)); - } - } From 06018b8bc0cb940d56c0529e634c60f5987fb4a7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 14 Jul 2015 23:16:36 +0900 Subject: [PATCH 21/25] Update description for CatalogUtil::addOrDropPartition --- .../src/main/java/org/apache/tajo/catalog/CatalogUtil.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index e4b551f47c..14db079d22 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -792,14 +792,15 @@ public static AlterTableDesc setProperty(String tableName, KeyValueSet params, A } /** - * Add partition or Drop partition + * Converts passed parameters to a AlterTableDesc. This method would be called when adding a partition or dropping + * a table. This creates AlterTableDesc that is a wrapper class for protocol buffer. * * @param tableName table name * @param columns partition column names * @param values partition values * @param location partition location * @param alterTableType ADD_PARTITION or DROP_PARTITION - * @return + * @return AlterTableDesc */ public static AlterTableDesc addOrDropPartition(String tableName, String[] columns, String[] values, String location, AlterTableType alterTableType) { From 5c213f2233be7b4ee5218fde3386d8bc9488c5ce Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Tue, 14 Jul 2015 23:48:12 +0900 Subject: [PATCH 22/25] Use PartitionKeyProto instead of PartitionKey --- .../org/apache/tajo/catalog/CatalogUtil.java | 20 +-- .../tajo/catalog/partition/PartitionDesc.java | 50 ++---- .../tajo/catalog/partition/PartitionKey.java | 147 ------------------ .../catalog/store/TestHiveCatalogStore.java | 10 +- .../org/apache/tajo/catalog/TestCatalog.java | 12 +- .../apache/tajo/master/exec/DDLExecutor.java | 4 +- 6 files changed, 39 insertions(+), 204 deletions(-) delete mode 100644 tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 14db079d22..ecf8890a22 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -24,11 +24,11 @@ import org.apache.tajo.DataTypeUtil; import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.partition.PartitionDesc; -import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; @@ -808,7 +808,7 @@ public static AlterTableDesc addOrDropPartition(String tableName, String[] colum alterTableDesc.setTableName(tableName); PartitionDesc partitionDesc = new PartitionDesc(); - Pair, String> pair = getPartitionKeyNamePair(columns, values); + Pair, String> pair = getPartitionKeyNamePair(columns, values); partitionDesc.setPartitionKeys(pair.getFirst()); partitionDesc.setPartitionName(pair.getSecond()); @@ -834,24 +834,24 @@ public static AlterTableDesc addOrDropPartition(String tableName, String[] colum * @param values partition values * @return partition key/value list and partition name */ - public static Pair, String> getPartitionKeyNamePair(String[] columns, String[] values) { - Pair, String> pair = null; - List partitionKeyList = TUtil.newList(); + public static Pair, String> getPartitionKeyNamePair(String[] columns, String[] values) { + Pair, String> pair = null; + List partitionKeyList = TUtil.newList(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < columns.length; i++) { - PartitionKey partitionKey = new PartitionKey(); - partitionKey.setColumnName(columns[i]); - partitionKey.setPartitionValue(values[i]); + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + builder.setColumnName(columns[i]); + builder.setPartitionValue(values[i]); if (i > 0) { sb.append("/"); } sb.append(columns[i]).append("=").append(values[i]); - partitionKeyList.add(partitionKey); + partitionKeyList.add(builder.build()); } - pair = new Pair, String>(partitionKeyList, sb.toString()); + pair = new Pair, String>(partitionKeyList, sb.toString()); return pair; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java index b6d883d150..7287fceb88 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionDesc.java @@ -22,13 +22,12 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; -import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; -import java.util.ArrayList; import java.util.List; /** @@ -55,36 +54,11 @@ */ public class PartitionDesc implements ProtoObject, Cloneable, GsonObject { @Expose protected String partitionName; - @Expose protected List partitionKeys; + @Expose protected List partitionKeys; @Expose protected String path; //optional private CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder(); - public PartitionDesc() { - } - - public PartitionDesc(PartitionDesc partition) { - this.partitionName = partition.partitionName; - this.partitionKeys = partition.partitionKeys; - this.path = partition.path; - } - - public PartitionDesc(CatalogProtos.PartitionDescProto proto) { - if(proto.hasPartitionName()) { - this.partitionName = proto.getPartitionName(); - } - - this.partitionKeys = new ArrayList(); - for(CatalogProtos.PartitionKeyProto keyProto : proto.getPartitionKeysList()) { - PartitionKey partitionKey = new PartitionKey(keyProto); - this.partitionKeys.add(partitionKey); - } - - if(proto.hasPath()) { - this.path = proto.getPath(); - } - } - public String getPartitionName() { return partitionName; } @@ -93,14 +67,6 @@ public void setPartitionName(String partitionName) { this.partitionName = partitionName; } - public List getPartitionKeys() { - return partitionKeys; - } - - public void setPartitionKeys(List partitionKeys) { - this.partitionKeys = partitionKeys; - } - public void setPath(String path) { this.path = path; } @@ -109,6 +75,14 @@ public String getPath() { return path; } + public List getPartitionKeys() { + return partitionKeys; + } + + public void setPartitionKeys(List partitionKeys) { + this.partitionKeys = partitionKeys; + } + public int hashCode() { return Objects.hashCode(partitionName, partitionKeys, path); } @@ -142,8 +116,8 @@ public CatalogProtos.PartitionDescProto getProto() { builder.clearPartitionKeys(); if (this.partitionKeys != null) { - for(PartitionKey partitionKey : this.partitionKeys) { - builder.addPartitionKeys(partitionKey.getProto()); + for(PartitionKeyProto partitionKey : this.partitionKeys) { + builder.addPartitionKeys(partitionKey); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java deleted file mode 100644 index 085598b7ec..0000000000 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/partition/PartitionKey.java +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.catalog.partition; - -import com.google.common.base.Objects; -import com.google.gson.annotations.Expose; -import org.apache.tajo.catalog.json.CatalogGsonHelper; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.common.ProtoObject; -import org.apache.tajo.json.GsonObject; -import org.apache.tajo.util.TUtil; - - -/** - * This presents column name and partition value pairs of column partitioned table. - * - * For example, consider you have a partitioned table as follows: - * - * create external table table1 (id text, name text) PARTITION BY COLUMN (dt text, phone text, - * gender text) USING RCFILE LOCATION '/tajo/data/table1'; - * - * Then, its data will be stored on HDFS as follows: - * - /tajo/data/table1/dt=20150301/phone=1300/gender=m - * - /tajo/data/table1/dt=20150301/phone=1300/gender=f - * - /tajo/data/table1/dt=20150302/phone=1500/gender=m - * - /tajo/data/table1/dt=20150302/phone=1500/gender=f - * - * In such as above, first directory can be presented with this class as follows: - * The first pair: column name = dt, partition value = 20150301 - * The second pair: column name = phone, partition value = 1300 - * The thris pair: column name = gender, partition value = m - * - */ -public class PartitionKey implements ProtoObject, Cloneable, GsonObject { - @Expose protected String columnName; // required - @Expose protected String partitionValue; // required - - private CatalogProtos.PartitionKeyProto.Builder builder = CatalogProtos.PartitionKeyProto.newBuilder(); - - public PartitionKey() { - } - - public PartitionKey(String columnName, String partitionValue) { - this.columnName = columnName; - this.partitionValue = partitionValue; - } - - public PartitionKey(PartitionKey partition) { - this.columnName = partition.columnName; - this.partitionValue = partition.partitionValue; - } - - public PartitionKey(CatalogProtos.PartitionKeyProto proto) { - if (proto.hasColumnName()) { - this.columnName = proto.getColumnName(); - } - if (proto.hasPartitionValue()) { - this.partitionValue = proto.getPartitionValue(); - } - } - - public String getPartitionValue() { - return partitionValue; - } - - public void setPartitionValue(String partitionValue) { - this.partitionValue = partitionValue; - } - - public String getColumnName() { - return columnName; - } - - public void setColumnName(String columnName) { - this.columnName = columnName; - } - - public int hashCode() { - return Objects.hashCode(partitionValue, columnName); - } - - public boolean equals(Object o) { - if (o instanceof PartitionKey) { - PartitionKey another = (PartitionKey) o; - return TUtil.checkEquals(columnName, another.columnName) && - TUtil.checkEquals(partitionValue, another.partitionValue); - } - return false; - } - - @Override - public CatalogProtos.PartitionKeyProto getProto() { - if (builder == null) { - builder = CatalogProtos.PartitionKeyProto.newBuilder(); - } - - if (this.columnName != null) { - builder.setColumnName(this.columnName); - } - - if (this.partitionValue != null) { - builder.setPartitionValue(this.partitionValue); - } - - return builder.build(); - } - - public String toString() { - StringBuilder sb = new StringBuilder("name: " + partitionValue); - return sb.toString(); - } - - @Override - public String toJson() { - return CatalogGsonHelper.toJson(this, PartitionKey.class); - } - - public static PartitionKey fromJson(String strVal) { - return strVal != null ? CatalogGsonHelper.fromJson(strVal, PartitionKey.class) : null; - } - - public Object clone() throws CloneNotSupportedException { - PartitionKey desc = (PartitionKey) super.clone(); - desc.builder = CatalogProtos.PartitionKeyProto.newBuilder(); - desc.partitionValue = partitionValue; - desc.columnName = columnName; - - return desc; - } - -} \ No newline at end of file diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index 946d271576..1a8282a83b 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -26,9 +26,9 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionDesc; -import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.storage.StorageConstants; @@ -286,11 +286,15 @@ private void testAddPartition(URI uri, String tableName, String partitionName) t PartitionDesc partitionDesc = new PartitionDesc(); partitionDesc.setPartitionName(partitionName); - List partitionKeyList = new ArrayList(); + List partitionKeyList = new ArrayList(); String[] partitionNames = partitionName.split("/"); for(int i = 0; i < partitionNames.length; i++) { String[] eachPartitionName = partitionNames[i].split("="); - partitionKeyList.add(new PartitionKey(eachPartitionName[0], eachPartitionName[1])); + + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + builder.setColumnName(eachPartitionName[0]); + builder.setPartitionValue(eachPartitionName[1]); + partitionKeyList.add(builder.build()); } partitionDesc.setPartitionKeys(partitionKeyList); partitionDesc.setPath(path.toString()); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index cbcec83f5a..2d52e9c395 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -26,13 +26,12 @@ import org.apache.tajo.catalog.exception.CatalogException; import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.partition.PartitionDesc; -import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.store.PostgreSQLStore; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.catalog.store.DerbyStore; import org.apache.tajo.catalog.store.MySQLStore; import org.apache.tajo.catalog.store.MariaDBStore; @@ -936,10 +935,15 @@ private void testAddPartition(String tableName, String partitionName) throws Exc String[] partitionNames = partitionName.split("/"); - List partitionKeyList = new ArrayList(); + List partitionKeyList = new ArrayList(); for(int i = 0; i < partitionNames.length; i++) { String columnName = partitionNames[i].split("=")[0]; - partitionKeyList.add(new PartitionKey(partitionNames[i], columnName)); + + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + builder.setColumnName(partitionNames[i]); + builder.setPartitionValue(columnName); + + partitionKeyList.add(builder.build()); } partitionDesc.setPartitionKeys(partitionKeyList); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index f49ea274f4..3e0b3204a8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -29,10 +29,10 @@ import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.exception.*; -import org.apache.tajo.catalog.partition.PartitionKey; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.TajoMaster; @@ -405,7 +405,7 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer Path partitionPath = null; TableDesc desc = null; - Pair, String> pair = null; + Pair, String> pair = null; CatalogProtos.PartitionDescProto partitionDescProto = null; if (alterTable.getAlterTableOpType() == AlterTableOpType.RENAME_TABLE From 8462870caf44d00c42871bf297ea20df1db9529e Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 15 Jul 2015 08:19:34 +0900 Subject: [PATCH 23/25] Update the message for NoSuchPartitionKeyException. --- .../tajo/catalog/exception/NoSuchPartitionKeyException.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java index c6bcd23793..94574dcde2 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionKeyException.java @@ -23,6 +23,7 @@ public class NoSuchPartitionKeyException extends RuntimeException { private static final long serialVersionUID = 277182608283894939L; public NoSuchPartitionKeyException(String tableName, String partitionKey) { - super(String.format("ERROR: partition column \"%s\" does not exist in \"%s\".", partitionKey, tableName)); + super(String.format("ERROR: \"%s\" column is not the partition key of \"%s\".", + partitionKey, tableName)); } } From b88fdb04be15cc4c2e0da7c37429d24e0ce9ef8a Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 15 Jul 2015 09:28:43 +0900 Subject: [PATCH 24/25] Fix the message for NoSuchPartitionException --- .../tajo/catalog/exception/NoSuchPartitionException.java | 4 ++-- .../results/TestTajoCli/testAlterTableAddDropPartition.result | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java index d0b266a87e..70e0d26e69 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/NoSuchPartitionException.java @@ -27,11 +27,11 @@ public NoSuchPartitionException(String message) { } public NoSuchPartitionException(String tableName, String partitionName) { - super(String.format("ERROR: \"%s\" does not exist in \"%s\".", partitionName, tableName)); + super(String.format("ERROR: \"%s\" is not the partition of \"%s\".", partitionName, tableName)); } public NoSuchPartitionException(String databaseName, String tableName, String partitionName) { - super(String.format("ERROR: \"%s\" does not exist in \"%s.%s\".", partitionName, databaseName, tableName)); + super(String.format("ERROR: \"%s\" is not the partition of \"%s.%s\".", partitionName, databaseName, tableName)); } } diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result index d5e0f65bca..c16c311dd6 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddDropPartition.result @@ -1,8 +1,8 @@ OK -ERROR: partition column "key2" does not exist in "default.testaltertableaddpartition". +ERROR: "key2" column is not the partition key of "default.testaltertableaddpartition". OK OK -ERROR: "key=0.1" does not exist in "testaltertableaddpartition". +ERROR: "key=0.1" is not the partition of "testaltertableaddpartition". OK OK OK From 9947ca02ff165b34a0b679363c316f62128f5efb Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 15 Jul 2015 17:05:45 +0900 Subject: [PATCH 25/25] Add AlreadyExistsAssumedPartitionDirectoryException --- ...stsAssumedPartitionDirectoryException.java | 28 +++++++++++++++++++ .../apache/tajo/master/exec/DDLExecutor.java | 20 +++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java new file mode 100644 index 0000000000..df13f824a6 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/exception/AlreadyExistsAssumedPartitionDirectoryException.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog.exception; + +public class AlreadyExistsAssumedPartitionDirectoryException extends RuntimeException { + + private static final long serialVersionUID = 277182608283894931L; + + public AlreadyExistsAssumedPartitionDirectoryException(String message) { + super(String.format("ERROR: There is a directory which is assumed to be a partitioned directory : %s", message)); + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 3e0b3204a8..af8d796214 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -379,8 +379,14 @@ public void truncateTable(final QueryContext queryContext, final TruncateTableNo } } + /** - * ALTER TABLE SET ... + * Execute alter table statement using catalog api. + * + * @param context + * @param queryContext + * @param alterTable + * @throws IOException */ public void alterTable(TajoMaster.MasterContext context, final QueryContext queryContext, final AlterTableNode alterTable) throws IOException { @@ -479,11 +485,21 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer alterTable.setLocation(partitionPath.toString()); } + FileSystem fs = partitionPath.getFileSystem(context.getConf()); + + // If there is a directory which was assumed to be a partitioned directory and users don't input another + // location, this will throw exception. + Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond()); + boolean result1 = fs.exists(assumedDirectory); + boolean result2 = fs.exists(partitionPath); + if (fs.exists(assumedDirectory) && !assumedDirectory.equals(partitionPath)) { + throw new AlreadyExistsAssumedPartitionDirectoryException(assumedDirectory.toString()); + } + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION)); // If the partition's path doesn't exist, this would make the directory by force. - FileSystem fs = partitionPath.getFileSystem(context.getConf()); if (!fs.exists(partitionPath)) { fs.mkdirs(partitionPath); }