From 5311779ef2c481ef4f815843c44c877df0106d90 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Thu, 24 Sep 2015 23:38:29 +0900 Subject: [PATCH 01/16] TAJO-1887: Disable the alter table add partition statement temporarily. --- .../tajo/catalog/store/HiveCatalogStore.java | 16 ++++++----- tajo-catalog/tajo-catalog-server/pom.xml | 12 +++++++++ .../tajo/catalog/store/AbstractDBStore.java | 20 +++++++------- .../tajo/catalog/CatalogTestingUtil.java | 27 +++++++++++++++---- .../org/apache/tajo/catalog/TestCatalog.java | 19 ++++++++----- tajo-core-tests/pom.xml | 11 +++++--- .../tajo/engine/query/TestAlterTable.java | 24 +++++++++++------ .../alter_table_add_partition_1.sql | 1 - .../alter_table_add_partition_2.sql | 1 - .../alter_table_add_partition_3.sql | 2 -- .../alter_table_add_partition_4.sql | 1 - .../alter_table_add_partition_5.sql | 1 - .../apache/tajo/parser/sql/SQLAnalyzer.java | 12 +++++---- .../org/apache/tajo/parser/sql/SQLParser.g4 | 1 - 14 files changed, 96 insertions(+), 52 deletions(-) delete mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_1.sql delete mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_2.sql delete mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_3.sql delete mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_4.sql delete mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_5.sql diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 72f7485ab2..2c317f98cf 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -624,13 +624,15 @@ public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescPro addNewColumn(databaseName, tableName, alterTableDescProto.getAddColumn()); break; case ADD_PARTITION: - partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); - partitionDesc = getPartition(databaseName, tableName, partitionName); - if(partitionDesc != null) { - throw new DuplicatePartitionException(partitionName); - } - addPartition(databaseName, tableName, alterTableDescProto.getPartitionDesc()); - break; + // Disable the alter table add partition statement temporarily at TAJO-1887 +// partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); +// partitionDesc = getPartition(databaseName, tableName, partitionName); +// if(partitionDesc != null) { +// throw new DuplicatePartitionException(partitionName); +// } +// addPartition(databaseName, tableName, alterTableDescProto.getPartitionDesc()); +// break; + throw new UnsupportedOperationException(); case DROP_PARTITION: partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); partitionDesc = getPartition(databaseName, tableName, partitionName); diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml index f00f8d30cd..ca79eed4a3 100644 --- a/tajo-catalog/tajo-catalog-server/pom.xml +++ b/tajo-catalog/tajo-catalog-server/pom.xml @@ -127,6 +127,18 @@ org.apache.maven.plugins maven-surefire-report-plugin + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 700ef1a9ae..066afc70b2 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1081,15 +1081,17 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) addNewColumn(tableId, alterTableDescProto.getAddColumn()); break; case ADD_PARTITION: - partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); - try { - // check if it exists - getPartition(databaseName, tableName, partitionName); - throw new DuplicatePartitionException(partitionName); - } catch (UndefinedPartitionException e) { - } - addPartition(tableId, alterTableDescProto.getPartitionDesc()); - break; + // Disable the alter table add partition statement temporarily at TAJO-1887 +// partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); +// try { +// // check if it exists +// getPartition(databaseName, tableName, partitionName); +// throw new DuplicatePartitionException(partitionName); +// } catch (UndefinedPartitionException e) { +// } +// addPartition(tableId, alterTableDescProto.getPartitionDesc()); +// break; + throw new UnsupportedOperationException(); case DROP_PARTITION: partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); partitionDesc = getPartition(databaseName, tableName, partitionName); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java index a4dd9c3cc2..6e54b7125d 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java @@ -20,13 +20,13 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.fs.Path; -import org.apache.tajo.TajoConstants; import org.apache.tajo.annotation.NotNull; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.catalog.store.*; @@ -36,6 +36,7 @@ import org.apache.tajo.exception.UnsupportedCatalogStore; import org.apache.tajo.util.KeyValueSet; +import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -143,7 +144,26 @@ public static PartitionDesc buildPartitionDesc(String partitionName) { partitionDesc.setPartitionName(partitionName); String[] partitionNames = partitionName.split("/"); + partitionDesc.setPartitionKeys(buildPartitionKeyProtos(partitionNames)); + partitionDesc.setPath("hdfs://xxx.com/warehouse/" + partitionName); + return partitionDesc; + } + + public static PartitionDescProto buildPartitionDescProto(String partitionName) { + return buildPartitionDescProto(partitionName, "hdfs://xxx.com/warehouse"); + } + + public static PartitionDescProto buildPartitionDescProto(String partitionName, String tablePath) { + PartitionDescProto.Builder partitionDescProto = PartitionDescProto.newBuilder(); + partitionDescProto.setPartitionName(partitionName); + String[] partitionNames = partitionName.split("/"); + partitionDescProto.addAllPartitionKeys(buildPartitionKeyProtos(partitionNames)); + partitionDescProto.setPath(tablePath + File.separator + partitionName); + return partitionDescProto.build(); + } + + private static List buildPartitionKeyProtos(String[] partitionNames) { List partitionKeyList = new ArrayList<>(); for(int i = 0; i < partitionNames.length; i++) { String [] splits = partitionNames[i].split("="); @@ -165,10 +185,7 @@ public static PartitionDesc buildPartitionDesc(String partitionName) { partitionKeyList.add(builder.build()); } - partitionDesc.setPartitionKeys(partitionKeyList); - - partitionDesc.setPath("hdfs://xxx.com/warehouse/" + partitionName); - return partitionDesc; + return partitionKeyList; } public static void prepareBaseData(CatalogService catalog, String testDir) throws Exception { diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 36f98cf017..35d955dfd3 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -928,13 +928,18 @@ private void testGetPartitionsByAlgebra(String databaseName, String tableName) t } private void testAddPartition(String tableName, String partitionName) throws Exception { - AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(tableName); - alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); - - alterTableDesc.setPartitionDesc(CatalogTestingUtil.buildPartitionDesc(partitionName)); - - catalog.alterTable(alterTableDesc); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// AlterTableDesc alterTableDesc = new AlterTableDesc(); +// alterTableDesc.setTableName(tableName); +// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); +// +// alterTableDesc.setPartitionDesc(CatalogTestingUtil.buildPartitionDesc(partitionName)); +// +// catalog.alterTable(alterTableDesc); + String[] splits = tableName.split("\\."); + List partitions = TUtil.newList(); + partitions.add(CatalogTestingUtil.buildPartitionDescProto(partitionName)); + catalog.addPartitions(splits[0], splits[1], partitions, true); String [] split = CatalogUtil.splitFQTableName(tableName); diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index 20b7378ef0..275c4334f1 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -152,10 +152,6 @@ org.apache.tajo tajo-catalog-client - - org.apache.tajo - tajo-catalog-server - org.apache.tajo tajo-storage-common @@ -194,6 +190,13 @@ test-jar test + + org.apache.tajo + tajo-catalog-server + test-jar + test + ${parent.version} + org.apache.hadoop diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 6262a7ac82..6cbb0a71f6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.catalog.CatalogTestingUtil; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -29,6 +30,7 @@ import org.apache.tajo.exception.UndefinedPartitionException; import org.apache.tajo.exception.UndefinedPartitionMethodException; import org.apache.tajo.exception.UndefinedTableException; +import org.apache.tajo.util.TUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -58,7 +60,7 @@ public final void testAlterTableColumnName() throws Exception { public final void testAlterTableAddNewColumn() throws Exception { List createdNames = executeDDL("table1_ddl.sql", "table1.tbl", "EFG"); executeDDL("alter_table_add_new_column_ddl.sql", null); - assertColumnExists(createdNames.get(0),"cool"); + assertColumnExists(createdNames.get(0), "cool"); } @Test @@ -89,8 +91,12 @@ public final void testAlterTableAddPartition() throws Exception { assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "col3"); assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(1).getSimpleName(), "col4"); - executeDDL("alter_table_add_partition1.sql", null); - executeDDL("alter_table_add_partition2.sql", null); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// executeDDL("alter_table_add_partition1.sql", null); +// executeDDL("alter_table_add_partition2.sql", null); + List targetPartitions = TUtil.newList(); + targetPartitions.add(CatalogTestingUtil.buildPartitionDescProto("col3=1/col4=2", retrieved.getUri().toString())); + catalog.addPartitions("TestAlterTable", "partitioned_table", targetPartitions, true); List partitions = catalog.getPartitionsOfTable("TestAlterTable", "partitioned_table"); assertNotNull(partitions); @@ -102,10 +108,11 @@ public final void testAlterTableAddPartition() throws Exception { assertEquals(partitions.get(0).getPartitionKeysList().get(1).getPartitionValue(), "2"); assertNotNull(partitions.get(0).getPath()); - Path partitionPath = new Path(partitions.get(0).getPath()); - FileSystem fs = partitionPath.getFileSystem(conf); - assertTrue(fs.exists(partitionPath)); - assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// Path partitionPath = new Path(partitions.get(0).getPath()); +// FileSystem fs = partitionPath.getFileSystem(conf); +// assertTrue(fs.exists(partitionPath)); +// assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); executeDDL("alter_table_drop_partition1.sql", null); executeDDL("alter_table_drop_partition2.sql", null); @@ -113,7 +120,8 @@ public final void testAlterTableAddPartition() throws Exception { partitions = catalog.getPartitionsOfTable("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 0); - assertFalse(fs.exists(partitionPath)); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// assertFalse(fs.exists(partitionPath)); catalog.dropTable(tableName); } diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_1.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_1.sql deleted file mode 100644 index 0f91b68dd7..0000000000 --- a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_1.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_2.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_2.sql deleted file mode 100644 index ee4df750fe..0000000000 --- a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_2.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx.com/warehouse/table1/col1=1/col2=2' \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_3.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_3.sql deleted file mode 100644 index 79d14266ce..0000000000 --- a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_3.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE table1 ADD PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) -LOCATION 'hdfs://xxx.com/warehouse/table1/col1=2015/col2=01/col3=11' \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_4.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_4.sql deleted file mode 100644 index bc22a1418f..0000000000 --- a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_4.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE table1 ADD PARTITION (col1 = 'TAJO' ) \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_5.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_5.sql deleted file mode 100644 index 127d9993c3..0000000000 --- a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_5.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE table1 ADD IF NOT EXISTS PARTITION (col1 = 'TAJO' ) \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java index 717366a88b..930c959458 100644 --- a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java @@ -1910,12 +1910,14 @@ public Expr visitAlter_table_statement(Alter_table_statementContext ctx) { } alterTable.setColumns(columns); alterTable.setValues(values); - if (ctx.LOCATION() != null) { - String path = stripQuote(ctx.path.getText()); - alterTable.setLocation(path); - } + // Disable the alter table add partition statement temporarily at TAJO-1887 +// if (ctx.LOCATION() != null) { +// String path = stripQuote(ctx.path.getText()); +// alterTable.setLocation(path); +// } +// alterTable.setIfNotExists(checkIfExist(ctx.if_not_exists())); + alterTable.setPurge(checkIfExist(ctx.PURGE())); - alterTable.setIfNotExists(checkIfExist(ctx.if_not_exists())); alterTable.setIfExists(checkIfExist(ctx.if_exists())); } diff --git a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 index e2693eae44..630f46bbd7 100644 --- a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 +++ b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 @@ -1622,7 +1622,6 @@ alter_table_statement : ALTER TABLE table_name RENAME TO table_name | ALTER TABLE table_name RENAME COLUMN column_name TO column_name | ALTER TABLE table_name ADD COLUMN field_element - | ALTER TABLE table_name ADD (if_not_exists)? PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (LOCATION path=Character_String_Literal)? | ALTER TABLE table_name DROP (if_exists)? PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (PURGE)? | ALTER TABLE table_name SET PROPERTY property_list | ALTER TABLE table_name REPAIR PARTITION From 96bbd418253a7e27a87432fdd72b171f9bce48d6 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 00:38:56 +0900 Subject: [PATCH 02/16] Fix unit test bugs --- .../org/apache/tajo/catalog/CatalogUtil.java | 48 ++++++++- .../catalog/store/TestHiveCatalogStore.java | 22 +++-- tajo-catalog/tajo-catalog-server/pom.xml | 12 --- .../tajo/catalog/CatalogTestingUtil.java | 49 ---------- .../org/apache/tajo/catalog/TestCatalog.java | 11 ++- .../TestCatalogAgainstCaseSensitivity.java | 48 +++++---- .../tajo/catalog/TestCatalogExceptions.java | 69 +++++++------ tajo-core-tests/pom.xml | 11 +-- .../tajo/engine/query/TestAlterTable.java | 3 +- .../apache/tajo/master/exec/DDLExecutor.java | 97 ++++++++++--------- 10 files changed, 194 insertions(+), 176 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 996bfaf733..3eeae125da 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -33,10 +33,10 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableIdentifierProto; +import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; -import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UndefinedOperatorException; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; @@ -44,6 +44,7 @@ import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; +import java.io.File; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -1043,4 +1044,49 @@ public int compare(String o1, String o2) { return originlSchema.getColumnId(o1) - originlSchema.getColumnId(o2); } } + + public static PartitionDesc buildPartitionDesc(String partitionName, String tablePath) { + PartitionDesc partitionDesc = new PartitionDesc(); + partitionDesc.setPartitionName(partitionName); + + String[] partitionNames = partitionName.split("/"); + partitionDesc.setPartitionKeys(buildPartitionKeyProtos(partitionNames)); + partitionDesc.setPath(tablePath + File.separator + partitionName); + return partitionDesc; + } + + public static PartitionDescProto buildPartitionDescProto(String partitionName, String tablePath) { + PartitionDescProto.Builder partitionDescProto = PartitionDescProto.newBuilder(); + partitionDescProto.setPartitionName(partitionName); + + String[] partitionNames = partitionName.split("/"); + partitionDescProto.addAllPartitionKeys(buildPartitionKeyProtos(partitionNames)); + partitionDescProto.setPath(tablePath + File.separator + partitionName); + return partitionDescProto.build(); + } + + private static List buildPartitionKeyProtos(String[] partitionNames) { + List partitionKeyList = new ArrayList<>(); + for(int i = 0; i < partitionNames.length; i++) { + String [] splits = partitionNames[i].split("="); + String columnName = "", partitionValue = ""; + if (splits.length == 2) { + columnName = splits[0]; + partitionValue = splits[1]; + } else if (splits.length == 1) { + if (partitionNames[i].charAt(0) == '=') { + partitionValue = splits[0]; + } else { + columnName = ""; + } + } + + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + builder.setColumnName(columnName); + builder.setPartitionValue(partitionValue); + partitionKeyList.add(builder.build()); + } + + return partitionKeyList; + } } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index f28eb21066..36a2d59dc2 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -261,12 +261,22 @@ public void testAddTableByPartition() throws Exception { assertEquals(partitionSchema.getColumn(i).getSimpleName(), partitionSchema1.getColumn(i).getSimpleName()); } - testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150101"); - testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150102"); - testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150101"); - testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150102"); - testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150101"); - testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150102"); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150101"); +// testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150102"); +// testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150101"); +// testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150102"); +// testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150101"); +// testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150102"); + List targetPartitions = TUtil.newList(); + String tablePath = table1.getUri().toString(); + targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=10/n_date=20150101", tablePath)); + targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=10/n_date=20150102", tablePath)); + targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=20/n_date=20150101", tablePath)); + targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=20/n_date=20150102", tablePath)); + targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=30/n_date=20150101", tablePath)); + targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=30/n_date=20150102", tablePath)); + store.addPartitions(DB_NAME, NATION, targetPartitions, true); List partitionNames = TUtil.newList(); partitionNames.add("n_nationkey=40/n_date=20150801"); diff --git a/tajo-catalog/tajo-catalog-server/pom.xml b/tajo-catalog/tajo-catalog-server/pom.xml index ca79eed4a3..f00f8d30cd 100644 --- a/tajo-catalog/tajo-catalog-server/pom.xml +++ b/tajo-catalog/tajo-catalog-server/pom.xml @@ -127,18 +127,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - - org.apache.maven.plugins - maven-jar-plugin - 2.4 - - - - test-jar - - - - diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java index 6e54b7125d..40da1f7d0f 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java @@ -139,55 +139,6 @@ private static String getCatalogURIScheme(Class claz } } - public static PartitionDesc buildPartitionDesc(String partitionName) { - PartitionDesc partitionDesc = new PartitionDesc(); - partitionDesc.setPartitionName(partitionName); - - String[] partitionNames = partitionName.split("/"); - partitionDesc.setPartitionKeys(buildPartitionKeyProtos(partitionNames)); - partitionDesc.setPath("hdfs://xxx.com/warehouse/" + partitionName); - return partitionDesc; - } - - public static PartitionDescProto buildPartitionDescProto(String partitionName) { - return buildPartitionDescProto(partitionName, "hdfs://xxx.com/warehouse"); - } - - public static PartitionDescProto buildPartitionDescProto(String partitionName, String tablePath) { - PartitionDescProto.Builder partitionDescProto = PartitionDescProto.newBuilder(); - partitionDescProto.setPartitionName(partitionName); - - String[] partitionNames = partitionName.split("/"); - partitionDescProto.addAllPartitionKeys(buildPartitionKeyProtos(partitionNames)); - partitionDescProto.setPath(tablePath + File.separator + partitionName); - return partitionDescProto.build(); - } - - private static List buildPartitionKeyProtos(String[] partitionNames) { - List partitionKeyList = new ArrayList<>(); - for(int i = 0; i < partitionNames.length; i++) { - String [] splits = partitionNames[i].split("="); - String columnName = "", partitionValue = ""; - if (splits.length == 2) { - columnName = splits[0]; - partitionValue = splits[1]; - } else if (splits.length == 1) { - if (partitionNames[i].charAt(0) == '=') { - partitionValue = splits[0]; - } else { - columnName = ""; - } - } - - PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); - builder.setColumnName(columnName); - builder.setPartitionValue(partitionValue); - partitionKeyList.add(builder.build()); - } - - return partitionKeyList; - } - public static void prepareBaseData(CatalogService catalog, String testDir) throws Exception { catalog.createTablespace("space1", "hdfs://xxx.com/warehouse"); catalog.createTablespace("SpAcE1", "hdfs://xxx.com/warehouse"); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 35d955dfd3..531b384e38 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -39,6 +39,7 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -813,8 +814,8 @@ public final void testAddAndDeleteTablePartitionByColumn() throws Exception { assertEquals(retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN); assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id"); - testAddPartition(tableName, "id=10/name=aaa"); - testAddPartition(tableName, "id=20/name=bbb"); + testAddPartition(tableName, "id=10/name=aaa", retrieved.getUri().toString()); + testAddPartition(tableName, "id=20/name=bbb", retrieved.getUri().toString()); List partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName); assertNotNull(partitions); @@ -927,7 +928,7 @@ private void testGetPartitionsByAlgebra(String databaseName, String tableName) t assertEquals(2, partitions.size()); } - private void testAddPartition(String tableName, String partitionName) throws Exception { + private void testAddPartition(String tableName, String partitionName, String tablePath) throws Exception { // Disable the alter table add partition statement temporarily at TAJO-1887 // AlterTableDesc alterTableDesc = new AlterTableDesc(); // alterTableDesc.setTableName(tableName); @@ -938,7 +939,7 @@ private void testAddPartition(String tableName, String partitionName) throws Exc // catalog.alterTable(alterTableDesc); String[] splits = tableName.split("\\."); List partitions = TUtil.newList(); - partitions.add(CatalogTestingUtil.buildPartitionDescProto(partitionName)); + partitions.add(CatalogUtil.buildPartitionDescProto(partitionName, tablePath)); catalog.addPartitions(splits[0], splits[1], partitions, true); String [] split = CatalogUtil.splitFQTableName(tableName); @@ -947,7 +948,7 @@ private void testAddPartition(String tableName, String partitionName) throws Exc assertNotNull(resultDesc); assertEquals(resultDesc.getPartitionName(), partitionName); - assertEquals(resultDesc.getPath(), "hdfs://xxx.com/warehouse/" + partitionName); + assertEquals(resultDesc.getPath(), tablePath + File.separator + partitionName); assertEquals(resultDesc.getPartitionKeysCount(), 2); } diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index 0ca6baa55a..1e36509152 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -27,6 +27,7 @@ import org.apache.tajo.exception.UndefinedPartitionException; import org.apache.tajo.exception.UndefinedTableException; import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.TUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -39,6 +40,7 @@ public class TestCatalogAgainstCaseSensitivity { static CatalogServer server; static CatalogService catalog; + final static String TEST_URI = "hdfs://xxx.com/warehouse"; @BeforeClass public static void setup() throws Exception { @@ -155,14 +157,18 @@ public void testTablePartition() throws Exception { assertTrue(catalog.existsTable("TestDatabase1", "testPartition1")); String partitionName = "DaTe=bBb/dAtE=AaA"; - PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); - - AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); - alterTableDesc.setPartitionDesc(partitionDesc); - alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); - - catalog.alterTable(alterTableDesc); + PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); + + // Disable the alter table add partition statement temporarily at TAJO-1887 +// AlterTableDesc alterTableDesc = new AlterTableDesc(); +// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); +// alterTableDesc.setPartitionDesc(partitionDesc); +// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); +// +// catalog.alterTable(alterTableDesc); + List targetPartitions = TUtil.newList(); + targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); + catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); PartitionDescProto resultDesc = catalog.getPartition("TestDatabase1", "TestPartition1", partitionName); @@ -173,14 +179,18 @@ public void testTablePartition() throws Exception { assertEquals(resultDesc.getPartitionKeysCount(), 2); partitionName = "DaTe=BbB/dAtE=aAa"; - partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); - - alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); - alterTableDesc.setPartitionDesc(partitionDesc); - alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); - - catalog.alterTable(alterTableDesc); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); +// +// alterTableDesc = new AlterTableDesc(); +// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); +// alterTableDesc.setPartitionDesc(partitionDesc); +// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); +// +// catalog.alterTable(alterTableDesc); + targetPartitions.clear(); + targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); + catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); resultDesc = catalog.getPartition("TestDatabase1", "TestPartition1", partitionName); @@ -221,9 +231,9 @@ public void testTablePartition() throws Exception { ////////////////////////////////////////////////////////////////////////////// partitionName = "DaTe=BbB/dAtE=aAa"; - partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); - alterTableDesc = new AlterTableDesc(); + AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.DROP_PARTITION); @@ -237,7 +247,7 @@ public void testTablePartition() throws Exception { assertNull(resultDesc); partitionName = "DaTe=bBb/dAtE=AaA"; - partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java index 6586cec808..c0ec3a402a 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java @@ -31,16 +31,19 @@ import org.apache.tajo.exception.*; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.TUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import java.net.URI; +import java.util.List; public class TestCatalogExceptions { static CatalogServer server; static CatalogService catalog; + final static String TEST_URI = "hdfs://xxx.com/warehouse"; @BeforeClass public static void setup() throws Exception { @@ -161,43 +164,55 @@ public void testUpdateTableStatsOfUndefinedTable() throws Exception { public void testAddPartitionWithWrongUri() throws Exception { // TODO: currently, wrong uri does not occur any exception. String partitionName = "DaTe=/=AaA"; - PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); - - AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); - alterTableDesc.setPartitionDesc(partitionDesc); - alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); - - catalog.alterTable(alterTableDesc); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, ""); +// +// AlterTableDesc alterTableDesc = new AlterTableDesc(); +// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); +// alterTableDesc.setPartitionDesc(partitionDesc); +// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); +// +// catalog.alterTable(alterTableDesc); + List targetPartitions = TUtil.newList(); + targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); + catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); } - @Test(expected = DuplicatePartitionException.class) + // Disable the alter table add partition statement temporarily at TAJO-1887 +// @Test(expected = DuplicatePartitionException.class) + @Test(expected = TajoInternalError.class) public void testAddDuplicatePartition() throws Exception { String partitionName = "DaTe=bBb/dAtE=AaA"; - PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); - - AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); - alterTableDesc.setPartitionDesc(partitionDesc); - alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); - - catalog.alterTable(alterTableDesc); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); +// AlterTableDesc alterTableDesc = new AlterTableDesc(); +// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); +// alterTableDesc.setPartitionDesc(partitionDesc); +// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); +// +// catalog.alterTable(alterTableDesc); + List targetPartitions = TUtil.newList(); + targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); + catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); partitionName = "DaTe=bBb/dAtE=AaA"; - partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); - - alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); - alterTableDesc.setPartitionDesc(partitionDesc); - alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); - - catalog.alterTable(alterTableDesc); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); +// +// alterTableDesc = new AlterTableDesc(); +// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); +// alterTableDesc.setPartitionDesc(partitionDesc); +// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); +// +// catalog.alterTable(alterTableDesc); + targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); + catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); } @Test(expected = UndefinedTableException.class) public void testAddPartitionToUndefinedTable() throws Exception { String partitionName = "DaTe=bBb/dAtE=AaA"; - PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "undefined")); @@ -210,7 +225,7 @@ public void testAddPartitionToUndefinedTable() throws Exception { @Test(expected = UndefinedPartitionException.class) public void testDropUndefinedPartition() throws Exception { String partitionName = "DaTe=undefined/dAtE=undefined"; - PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); diff --git a/tajo-core-tests/pom.xml b/tajo-core-tests/pom.xml index 275c4334f1..20b7378ef0 100644 --- a/tajo-core-tests/pom.xml +++ b/tajo-core-tests/pom.xml @@ -152,6 +152,10 @@ org.apache.tajo tajo-catalog-client + + org.apache.tajo + tajo-catalog-server + org.apache.tajo tajo-storage-common @@ -190,13 +194,6 @@ test-jar test - - org.apache.tajo - tajo-catalog-server - test-jar - test - ${parent.version} - org.apache.hadoop diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 6cbb0a71f6..25ce4f3633 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -22,7 +22,6 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; -import org.apache.tajo.catalog.CatalogTestingUtil; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -95,7 +94,7 @@ public final void testAlterTableAddPartition() throws Exception { // executeDDL("alter_table_add_partition1.sql", null); // executeDDL("alter_table_add_partition2.sql", null); List targetPartitions = TUtil.newList(); - targetPartitions.add(CatalogTestingUtil.buildPartitionDescProto("col3=1/col4=2", retrieved.getUri().toString())); + targetPartitions.add(CatalogUtil.buildPartitionDescProto("col3=1/col4=2", retrieved.getUri().toString())); catalog.addPartitions("TestAlterTable", "partitioned_table", targetPartitions, true); List partitions = catalog.getPartitionsOfTable("TestAlterTable", "partitioned_table"); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 57d01aa4a0..207fd0efde 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -477,54 +477,55 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer .SET_PROPERTY)); break; case ADD_PARTITION: - pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), alterTable.getPartitionValues()); - ensureColumnPartitionKeys(qualifiedName, alterTable.getPartitionColumns()); - - // checking a duplicated partition - boolean duplicatedPartition = true; - try { - catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); - } catch (UndefinedPartitionException e) { - duplicatedPartition = false; - } - - if (duplicatedPartition && !alterTable.isIfNotExists()) { - throw new DuplicatePartitionException(pair.getSecond()); - } else if (!duplicatedPartition) { - if (alterTable.getLocation() != null) { - partitionPath = new Path(alterTable.getLocation()); - } else { - // If location is not specified, the partition's location will be set using the table location. - partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); - alterTable.setLocation(partitionPath.toString()); - } - - FileSystem fs = partitionPath.getFileSystem(context.getConf()); - - // If there is a directory which was assumed to be a partitioned directory and users don't input another - // location, this will throw exception. - Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond()); - - if (fs.exists(assumedDirectory) && !assumedDirectory.equals(partitionPath)) { - throw new AmbiguousPartitionDirectoryExistException(assumedDirectory.toString()); - } - - long numBytes = 0L; - if (fs.exists(partitionPath)) { - ContentSummary summary = fs.getContentSummary(partitionPath); - numBytes = summary.getLength(); - } - - catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), - alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION, numBytes)); - - // If the partition's path doesn't exist, this would make the directory by force. - if (!fs.exists(partitionPath)) { - fs.mkdirs(partitionPath); - } - } - - break; +// pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), alterTable.getPartitionValues()); +// ensureColumnPartitionKeys(qualifiedName, alterTable.getPartitionColumns()); +// +// // checking a duplicated partition +// boolean duplicatedPartition = true; +// try { +// catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); +// } catch (UndefinedPartitionException e) { +// duplicatedPartition = false; +// } +// +// if (duplicatedPartition && !alterTable.isIfNotExists()) { +// throw new DuplicatePartitionException(pair.getSecond()); +// } else if (!duplicatedPartition) { +// if (alterTable.getLocation() != null) { +// partitionPath = new Path(alterTable.getLocation()); +// } else { +// // If location is not specified, the partition's location will be set using the table location. +// partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); +// alterTable.setLocation(partitionPath.toString()); +// } +// +// FileSystem fs = partitionPath.getFileSystem(context.getConf()); +// +// // If there is a directory which was assumed to be a partitioned directory and users don't input another +// // location, this will throw exception. +// Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond()); +// +// if (fs.exists(assumedDirectory) && !assumedDirectory.equals(partitionPath)) { +// throw new AmbiguousPartitionDirectoryExistException(assumedDirectory.toString()); +// } +// +// long numBytes = 0L; +// if (fs.exists(partitionPath)) { +// ContentSummary summary = fs.getContentSummary(partitionPath); +// numBytes = summary.getLength(); +// } +// +// catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), +// alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION, numBytes)); +// +// // If the partition's path doesn't exist, this would make the directory by force. +// if (!fs.exists(partitionPath)) { +// fs.mkdirs(partitionPath); +// } +// } +// +// break; + throw new TajoRuntimeException(new UnsupportedException("ADD PARTITION")); case DROP_PARTITION: ensureColumnPartitionKeys(qualifiedName, alterTable.getPartitionColumns()); pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), alterTable.getPartitionValues()); From a10985abca6df4714284022cbd81d6f754f49061 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 00:44:07 +0900 Subject: [PATCH 03/16] Fix an unit test bug for JDBC --- tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index d29291e5a2..209a82dad8 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -590,6 +590,8 @@ public void testSortWithDateTime() throws Exception { } + // Disable the alter table add partition statement temporarily at TAJO-1887 + /* @Test public void testAlterTableAddPartition() throws Exception { Statement stmt = null; @@ -629,6 +631,7 @@ public void testAlterTableAddPartition() throws Exception { } } } + */ @Test public void testMaxRows() throws Exception { From 9dc0cee5175439f3832b271a7a67b217bede6f2e Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 00:59:44 +0900 Subject: [PATCH 04/16] Update add partition statement to repair partition statement --- .../main/java/org/apache/tajo/cli/tools/TajoDump.java | 11 +++++++---- .../java/org/apache/tajo/cli/tools/TestTajoDump.java | 10 ++++++---- .../results/TestTajoDump/testPartitionsDump.result | 4 ++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index c21e2949c5..0ef98a5b79 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -192,10 +192,13 @@ private static void dumpDatabase(TajoClient client, String databaseName, PrintWr writer.write("--\n"); writer.write(String.format("-- Table Partitions: %s%n", tableName)); writer.write("--\n"); - List partitionProtos = client.getPartitionsOfTable(fqName); - for (PartitionDescProto eachPartitionProto : partitionProtos) { - writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto)); - } + // Disable the alter table add partition statement temporarily at TAJO-1887 +// List partitionProtos = client.getPartitionsOfTable(fqName); +// for (PartitionDescProto eachPartitionProto : partitionProtos) { +// writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto)); +// } + writer.write(String.format("ALTER TABLE %s REPAIR PARTITION%n", + CatalogUtil.denormalizeIdentifier(databaseName) + "." + CatalogUtil.denormalizeIdentifier(tableName))); writer.write("\n\n"); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java index 365aca6f2d..18600e75ac 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -113,16 +113,18 @@ public void testPartitionsDump() throws Exception { + " partition by column(\"col3\" int4, \"col4\" int4)" ); - executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"" + - " ADD PARTITION (\"col3\" = 1 , \"col4\" = 2)"); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"" + +// " ADD PARTITION (\"col3\" = 1 , \"col4\" = 2)"); executeString("create table \"" + getCurrentDatabase() + "\".\"TableName4\"" + " (\"col1\" int4, \"col2\" int4) " + " partition by column(\"col3\" TEXT, \"col4\" date)" ); - executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"" + - " ADD PARTITION (\"col3\" = 'tajo' , \"col4\" = '2015-09-01')"); + // Disable the alter table add partition statement temporarily at TAJO-1887 +// executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"" + +// " ADD PARTITION (\"col3\" = 'tajo' , \"col4\" = '2015-09-01')"); try { UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result index 696bb191cc..73db854bff 100644 --- a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result +++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result @@ -17,7 +17,7 @@ CREATE TABLE "TestTajoDump"."TableName3" (col1 INT4, col2 INT4) USING TEXT WITH -- -- Table Partitions: TableName3 -- -ALTER TABLE "TestTajoDump"."TableName3" ADD IF NOT EXISTS PARTITION (col3=1,col4=2) LOCATION '${partition.path1}/col3=1/col4=2'; +ALTER TABLE "TestTajoDump"."TableName3" REPAIR PARTITION @@ -30,4 +30,4 @@ CREATE TABLE "TestTajoDump"."TableName4" (col1 INT4, col2 INT4) USING TEXT WITH -- -- Table Partitions: TableName4 -- -ALTER TABLE "TestTajoDump"."TableName4" ADD IF NOT EXISTS PARTITION (col3='tajo',col4='2015-09-01') LOCATION '${partition.path2}/col3=tajo/col4=2015-09-01'; \ No newline at end of file +ALTER TABLE "TestTajoDump"."TableName4" REPAIR PARTITION \ No newline at end of file From dee2d9330f87287e1807e58dc91c2072bac96a87 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 01:05:26 +0900 Subject: [PATCH 05/16] Disable unit tests for add partition --- .../src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index da51aedd9b..a10e290f8f 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -473,6 +473,8 @@ public void testNonForwardQueryPause() throws Exception { } } + // Disable the alter table add partition statement temporarily at TAJO-1887 +/* @Test public void testAlterTableAddDropPartition() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition"); @@ -503,6 +505,7 @@ public void testAlterTableAddDropPartition() throws Exception { String consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); } +*/ public static class TajoCliOutputTestFormatter extends DefaultTajoCliOutputFormatter { @Override From 8fb9b5a500505d7a295efc92bc1d32914ce1d640 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 01:28:56 +0900 Subject: [PATCH 06/16] Fix unit test bugs for logical plan --- .../engine/planner/TestLogicalPlanner.java | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 5a0bd94f9b..400c3915fb 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -1329,8 +1329,33 @@ public final void testAddPartitionAndDropPartition() throws TajoException { QueryContext qc = new QueryContext(util.getConfiguration(), session); + // Disable the alter table add partition statement temporarily at TAJO-1887 // Testing alter table add partition - Expr expr = sqlAnalyzer.parse(ALTER_PARTITIONS[0]); +// Expr expr = sqlAnalyzer.parse(ALTER_PARTITIONS[0]); +// LogicalPlan rootNode = planner.createPlan(qc, expr); +// LogicalNode plan = rootNode.getRootBlock().getRoot(); +// testJsonSerDerObject(plan); +// assertEquals(NodeType.ROOT, plan.getType()); +// LogicalRootNode root = (LogicalRootNode) plan; +// assertEquals(NodeType.ALTER_TABLE, root.getChild().getType()); +// +// AlterTableNode alterTableNode = root.getChild(); +// +// assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); +// +// assertEquals(alterTableNode.getPartitionColumns().length, 2); +// assertEquals(alterTableNode.getPartitionValues().length, 2); +// +// assertEquals(alterTableNode.getPartitionColumns()[0], "col1"); +// assertEquals(alterTableNode.getPartitionColumns()[1], "col2"); +// +// assertEquals(alterTableNode.getPartitionValues()[0], "1"); +// assertEquals(alterTableNode.getPartitionValues()[1], "2"); +// +// assertEquals(alterTableNode.getLocation(), "hdfs://xxx.com/warehouse/partitioned_table/col1=1/col2=2"); + + // Testing alter table drop partition + Expr expr = sqlAnalyzer.parse(ALTER_PARTITIONS[1]); LogicalPlan rootNode = planner.createPlan(qc, expr); LogicalNode plan = rootNode.getRootBlock().getRoot(); testJsonSerDerObject(plan); @@ -1340,30 +1365,6 @@ public final void testAddPartitionAndDropPartition() throws TajoException { AlterTableNode alterTableNode = root.getChild(); - assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); - - assertEquals(alterTableNode.getPartitionColumns().length, 2); - assertEquals(alterTableNode.getPartitionValues().length, 2); - - assertEquals(alterTableNode.getPartitionColumns()[0], "col1"); - assertEquals(alterTableNode.getPartitionColumns()[1], "col2"); - - assertEquals(alterTableNode.getPartitionValues()[0], "1"); - assertEquals(alterTableNode.getPartitionValues()[1], "2"); - - assertEquals(alterTableNode.getLocation(), "hdfs://xxx.com/warehouse/partitioned_table/col1=1/col2=2"); - - // Testing alter table drop partition - expr = sqlAnalyzer.parse(ALTER_PARTITIONS[1]); - rootNode = planner.createPlan(qc, expr); - plan = rootNode.getRootBlock().getRoot(); - testJsonSerDerObject(plan); - assertEquals(NodeType.ROOT, plan.getType()); - root = (LogicalRootNode) plan; - assertEquals(NodeType.ALTER_TABLE, root.getChild().getType()); - - alterTableNode = root.getChild(); - assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.DROP_PARTITION); assertEquals(alterTableNode.getPartitionColumns().length, 3); From 15d59e2ac82342e186fe46fe9de667c808847481 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 11:33:36 +0900 Subject: [PATCH 07/16] Use verification rule instead of than commenting out --- .../tajo/catalog/store/HiveCatalogStore.java | 16 ++- .../catalog/store/TestHiveCatalogStore.java | 25 ++--- .../apache/tajo/catalog/CatalogServer.java | 8 +- .../tajo/catalog/store/AbstractDBStore.java | 20 ++-- .../tajo/catalog/CatalogTestingUtil.java | 36 ++++++- .../org/apache/tajo/catalog/TestCatalog.java | 30 +++--- .../TestCatalogAgainstCaseSensitivity.java | 51 ++++------ .../tajo/catalog/TestCatalogExceptions.java | 74 ++++++-------- .../apache/tajo/cli/tools/TestTajoDump.java | 5 +- .../engine/planner/TestLogicalPlanner.java | 54 +++++------ .../tajo/engine/query/TestAlterTable.java | 26 ++--- .../tajo/parser/sql/TestSQLAnalyzer.java | 7 +- .../alter_table_add_partition_1.sql | 1 + .../alter_table_add_partition_2.sql | 1 + .../alter_table_add_partition_3.sql | 2 + .../alter_table_add_partition_4.sql | 1 + .../alter_table_add_partition_5.sql | 1 + .../apache/tajo/master/exec/DDLExecutor.java | 97 +++++++++---------- .../apache/tajo/parser/sql/SQLAnalyzer.java | 12 +-- .../plan/verifier/PreLogicalPlanVerifier.java | 10 ++ .../org/apache/tajo/parser/sql/SQLParser.g4 | 1 + 21 files changed, 245 insertions(+), 233 deletions(-) create mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_1.sql create mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_2.sql create mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_3.sql create mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_4.sql create mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_5.sql diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 2c317f98cf..72f7485ab2 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -624,15 +624,13 @@ public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescPro addNewColumn(databaseName, tableName, alterTableDescProto.getAddColumn()); break; case ADD_PARTITION: - // Disable the alter table add partition statement temporarily at TAJO-1887 -// partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); -// partitionDesc = getPartition(databaseName, tableName, partitionName); -// if(partitionDesc != null) { -// throw new DuplicatePartitionException(partitionName); -// } -// addPartition(databaseName, tableName, alterTableDescProto.getPartitionDesc()); -// break; - throw new UnsupportedOperationException(); + partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); + partitionDesc = getPartition(databaseName, tableName, partitionName); + if(partitionDesc != null) { + throw new DuplicatePartitionException(partitionName); + } + addPartition(databaseName, tableName, alterTableDescProto.getPartitionDesc()); + break; case DROP_PARTITION: partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); partitionDesc = getPartition(databaseName, tableName, partitionName); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index 36a2d59dc2..9854e951ed 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -219,7 +219,8 @@ public void testTableWithNullValue() throws Exception { } - @Test + // TODO: This should be added at TAJO-1891 +// @Test public void testAddTableByPartition() throws Exception { TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); @@ -261,22 +262,12 @@ public void testAddTableByPartition() throws Exception { assertEquals(partitionSchema.getColumn(i).getSimpleName(), partitionSchema1.getColumn(i).getSimpleName()); } - // Disable the alter table add partition statement temporarily at TAJO-1887 -// testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150101"); -// testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150102"); -// testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150101"); -// testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150102"); -// testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150101"); -// testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150102"); - List targetPartitions = TUtil.newList(); - String tablePath = table1.getUri().toString(); - targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=10/n_date=20150101", tablePath)); - targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=10/n_date=20150102", tablePath)); - targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=20/n_date=20150101", tablePath)); - targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=20/n_date=20150102", tablePath)); - targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=30/n_date=20150101", tablePath)); - targetPartitions.add(CatalogUtil.buildPartitionDescProto("n_nationkey=30/n_date=20150102", tablePath)); - store.addPartitions(DB_NAME, NATION, targetPartitions, true); + testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150101"); + testAddPartition(table1.getUri(), NATION, "n_nationkey=10/n_date=20150102"); + testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150101"); + testAddPartition(table1.getUri(), NATION, "n_nationkey=20/n_date=20150102"); + testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150101"); + testAddPartition(table1.getUri(), NATION, "n_nationkey=30/n_date=20150102"); List partitionNames = TUtil.newList(); partitionNames.add("n_nationkey=40/n_date=20150801"); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 44eec84fbc..880687dbc4 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -36,6 +36,7 @@ import org.apache.tajo.catalog.CatalogProtocol.*; import org.apache.tajo.catalog.dictionary.InfoSchemaMetadataDictionary; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.error.Errors; import org.apache.tajo.exception.*; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.catalog.store.CatalogStore; @@ -457,7 +458,12 @@ public ReturnState alterTable(RpcController controller, AlterTableDescProto prot if (metaDictionary.isSystemDatabase(split[0])) { return errInsufficientPrivilege("alter a table in database '" + split[0] + "'"); } - + + // TODO: This should be removed at TAJO-1891 + if (proto.getAlterTableType() == CatalogProtos.AlterTableType.ADD_PARTITION) { + return errFeatureNotSupported("ADD PARTTIION"); + } + wlock.lock(); try { diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 066afc70b2..700ef1a9ae 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -1081,17 +1081,15 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) addNewColumn(tableId, alterTableDescProto.getAddColumn()); break; case ADD_PARTITION: - // Disable the alter table add partition statement temporarily at TAJO-1887 -// partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); -// try { -// // check if it exists -// getPartition(databaseName, tableName, partitionName); -// throw new DuplicatePartitionException(partitionName); -// } catch (UndefinedPartitionException e) { -// } -// addPartition(tableId, alterTableDescProto.getPartitionDesc()); -// break; - throw new UnsupportedOperationException(); + partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); + try { + // check if it exists + getPartition(databaseName, tableName, partitionName); + throw new DuplicatePartitionException(partitionName); + } catch (UndefinedPartitionException e) { + } + addPartition(tableId, alterTableDescProto.getPartitionDesc()); + break; case DROP_PARTITION: partitionName = alterTableDescProto.getPartitionDesc().getPartitionName(); partitionDesc = getPartition(databaseName, tableName, partitionName); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java index 40da1f7d0f..a4dd9c3cc2 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java @@ -20,13 +20,13 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.fs.Path; +import org.apache.tajo.TajoConstants; import org.apache.tajo.annotation.NotNull; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod; -import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.catalog.store.*; @@ -36,7 +36,6 @@ import org.apache.tajo.exception.UnsupportedCatalogStore; import org.apache.tajo.util.KeyValueSet; -import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -139,6 +138,39 @@ private static String getCatalogURIScheme(Class claz } } + public static PartitionDesc buildPartitionDesc(String partitionName) { + PartitionDesc partitionDesc = new PartitionDesc(); + partitionDesc.setPartitionName(partitionName); + + String[] partitionNames = partitionName.split("/"); + + List partitionKeyList = new ArrayList<>(); + for(int i = 0; i < partitionNames.length; i++) { + String [] splits = partitionNames[i].split("="); + String columnName = "", partitionValue = ""; + if (splits.length == 2) { + columnName = splits[0]; + partitionValue = splits[1]; + } else if (splits.length == 1) { + if (partitionNames[i].charAt(0) == '=') { + partitionValue = splits[0]; + } else { + columnName = ""; + } + } + + PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); + builder.setColumnName(columnName); + builder.setPartitionValue(partitionValue); + partitionKeyList.add(builder.build()); + } + + partitionDesc.setPartitionKeys(partitionKeyList); + + partitionDesc.setPath("hdfs://xxx.com/warehouse/" + partitionName); + return partitionDesc; + } + public static void prepareBaseData(CatalogService catalog, String testDir) throws Exception { catalog.createTablespace("space1", "hdfs://xxx.com/warehouse"); catalog.createTablespace("SpAcE1", "hdfs://xxx.com/warehouse"); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 531b384e38..606f39a382 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -39,7 +39,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -778,7 +777,7 @@ public final void testAddAndDeleteTablePartitionByRange() throws Exception { assertFalse(catalog.existsTable(tableName)); } - @Test + // TODO: This should be added at TAJO-1891 public final void testAddAndDeleteTablePartitionByColumn() throws Exception { Schema schema = new Schema(); schema.addColumn("id", Type.INT4) @@ -814,8 +813,8 @@ public final void testAddAndDeleteTablePartitionByColumn() throws Exception { assertEquals(retrieved.getPartitionMethod().getPartitionType(), CatalogProtos.PartitionType.COLUMN); assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "id"); - testAddPartition(tableName, "id=10/name=aaa", retrieved.getUri().toString()); - testAddPartition(tableName, "id=20/name=bbb", retrieved.getUri().toString()); + testAddPartition(tableName, "id=10/name=aaa"); + testAddPartition(tableName, "id=20/name=bbb"); List partitions = catalog.getPartitionsOfTable(DEFAULT_DATABASE_NAME, simpleTableName); assertNotNull(partitions); @@ -928,19 +927,14 @@ private void testGetPartitionsByAlgebra(String databaseName, String tableName) t assertEquals(2, partitions.size()); } - private void testAddPartition(String tableName, String partitionName, String tablePath) throws Exception { - // Disable the alter table add partition statement temporarily at TAJO-1887 -// AlterTableDesc alterTableDesc = new AlterTableDesc(); -// alterTableDesc.setTableName(tableName); -// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); -// -// alterTableDesc.setPartitionDesc(CatalogTestingUtil.buildPartitionDesc(partitionName)); -// -// catalog.alterTable(alterTableDesc); - String[] splits = tableName.split("\\."); - List partitions = TUtil.newList(); - partitions.add(CatalogUtil.buildPartitionDescProto(partitionName, tablePath)); - catalog.addPartitions(splits[0], splits[1], partitions, true); + private void testAddPartition(String tableName, String partitionName) throws Exception { + AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(tableName); + alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); + + alterTableDesc.setPartitionDesc(CatalogTestingUtil.buildPartitionDesc(partitionName)); + + catalog.alterTable(alterTableDesc); String [] split = CatalogUtil.splitFQTableName(tableName); @@ -948,7 +942,7 @@ private void testAddPartition(String tableName, String partitionName, String tab assertNotNull(resultDesc); assertEquals(resultDesc.getPartitionName(), partitionName); - assertEquals(resultDesc.getPath(), tablePath + File.separator + partitionName); + assertEquals(resultDesc.getPath(), "hdfs://xxx.com/warehouse/" + partitionName); assertEquals(resultDesc.getPartitionKeysCount(), 2); } diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index 1e36509152..24c0abfa94 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -27,7 +27,6 @@ import org.apache.tajo.exception.UndefinedPartitionException; import org.apache.tajo.exception.UndefinedTableException; import org.apache.tajo.util.CommonTestingUtil; -import org.apache.tajo.util.TUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -40,7 +39,6 @@ public class TestCatalogAgainstCaseSensitivity { static CatalogServer server; static CatalogService catalog; - final static String TEST_URI = "hdfs://xxx.com/warehouse"; @BeforeClass public static void setup() throws Exception { @@ -147,7 +145,8 @@ public void testTable() throws Exception { assertEquals(5000, stats.getNumRows().longValue()); } - @Test + // TODO: This should be added at TAJO-1891 +// @Test public void testTablePartition() throws Exception { ////////////////////////////////////////////////////////////////////////////// // Test add partition @@ -157,18 +156,14 @@ public void testTablePartition() throws Exception { assertTrue(catalog.existsTable("TestDatabase1", "testPartition1")); String partitionName = "DaTe=bBb/dAtE=AaA"; - PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); - - // Disable the alter table add partition statement temporarily at TAJO-1887 -// AlterTableDesc alterTableDesc = new AlterTableDesc(); -// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); -// alterTableDesc.setPartitionDesc(partitionDesc); -// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); -// -// catalog.alterTable(alterTableDesc); - List targetPartitions = TUtil.newList(); - targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); - catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); + PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + + AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); + + catalog.alterTable(alterTableDesc); PartitionDescProto resultDesc = catalog.getPartition("TestDatabase1", "TestPartition1", partitionName); @@ -179,18 +174,14 @@ public void testTablePartition() throws Exception { assertEquals(resultDesc.getPartitionKeysCount(), 2); partitionName = "DaTe=BbB/dAtE=aAa"; - // Disable the alter table add partition statement temporarily at TAJO-1887 -// partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); -// -// alterTableDesc = new AlterTableDesc(); -// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); -// alterTableDesc.setPartitionDesc(partitionDesc); -// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); -// -// catalog.alterTable(alterTableDesc); - targetPartitions.clear(); - targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); - catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); + partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + + alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); + + catalog.alterTable(alterTableDesc); resultDesc = catalog.getPartition("TestDatabase1", "TestPartition1", partitionName); @@ -231,9 +222,9 @@ public void testTablePartition() throws Exception { ////////////////////////////////////////////////////////////////////////////// partitionName = "DaTe=BbB/dAtE=aAa"; - partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); + partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); - AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.DROP_PARTITION); @@ -247,7 +238,7 @@ public void testTablePartition() throws Exception { assertNull(resultDesc); partitionName = "DaTe=bBb/dAtE=AaA"; - partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); + partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java index c0ec3a402a..e27f0c067a 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java @@ -31,19 +31,16 @@ import org.apache.tajo.exception.*; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.TUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import java.net.URI; -import java.util.List; public class TestCatalogExceptions { static CatalogServer server; static CatalogService catalog; - final static String TEST_URI = "hdfs://xxx.com/warehouse"; @BeforeClass public static void setup() throws Exception { @@ -160,59 +157,50 @@ public void testUpdateTableStatsOfUndefinedTable() throws Exception { build()); } - @Test + // TODO: This should be added at TAJO-1891 +// @Test public void testAddPartitionWithWrongUri() throws Exception { // TODO: currently, wrong uri does not occur any exception. String partitionName = "DaTe=/=AaA"; - // Disable the alter table add partition statement temporarily at TAJO-1887 -// PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, ""); -// -// AlterTableDesc alterTableDesc = new AlterTableDesc(); -// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); -// alterTableDesc.setPartitionDesc(partitionDesc); -// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); -// -// catalog.alterTable(alterTableDesc); - List targetPartitions = TUtil.newList(); - targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); - catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); + PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + + AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); + + catalog.alterTable(alterTableDesc); } - // Disable the alter table add partition statement temporarily at TAJO-1887 + // TODO: This should be added at TAJO-1891 // @Test(expected = DuplicatePartitionException.class) - @Test(expected = TajoInternalError.class) public void testAddDuplicatePartition() throws Exception { String partitionName = "DaTe=bBb/dAtE=AaA"; - // Disable the alter table add partition statement temporarily at TAJO-1887 -// PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); -// AlterTableDesc alterTableDesc = new AlterTableDesc(); -// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); -// alterTableDesc.setPartitionDesc(partitionDesc); -// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); -// -// catalog.alterTable(alterTableDesc); - List targetPartitions = TUtil.newList(); - targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); - catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); + PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + + AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); + + catalog.alterTable(alterTableDesc); partitionName = "DaTe=bBb/dAtE=AaA"; - // Disable the alter table add partition statement temporarily at TAJO-1887 -// partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); -// -// alterTableDesc = new AlterTableDesc(); -// alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); -// alterTableDesc.setPartitionDesc(partitionDesc); -// alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); -// -// catalog.alterTable(alterTableDesc); - targetPartitions.add(CatalogUtil.buildPartitionDescProto(partitionName, TEST_URI)); - catalog.addPartitions("TestDatabase1", "TestPartition1", targetPartitions, true); + partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + + alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); + + catalog.alterTable(alterTableDesc); } - @Test(expected = UndefinedTableException.class) + // TODO: This should be added at TAJO-1891 +// @Test(expected = UndefinedTableException.class) public void testAddPartitionToUndefinedTable() throws Exception { String partitionName = "DaTe=bBb/dAtE=AaA"; - PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); + PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "undefined")); @@ -225,7 +213,7 @@ public void testAddPartitionToUndefinedTable() throws Exception { @Test(expected = UndefinedPartitionException.class) public void testDropUndefinedPartition() throws Exception { String partitionName = "DaTe=undefined/dAtE=undefined"; - PartitionDesc partitionDesc = CatalogUtil.buildPartitionDesc(partitionName, TEST_URI); + PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java index 18600e75ac..0ae9b3e39a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -113,16 +113,17 @@ public void testPartitionsDump() throws Exception { + " partition by column(\"col3\" int4, \"col4\" int4)" ); - // Disable the alter table add partition statement temporarily at TAJO-1887 + // TODO: This should be added at TAJO-1891 // executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"" + // " ADD PARTITION (\"col3\" = 1 , \"col4\" = 2)"); + // TODO: This should be removed at TAJO-1891 executeString("create table \"" + getCurrentDatabase() + "\".\"TableName4\"" + " (\"col1\" int4, \"col2\" int4) " + " partition by column(\"col3\" TEXT, \"col4\" date)" ); - // Disable the alter table add partition statement temporarily at TAJO-1887 + // TODO: This should be added at TAJO-1891 // executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"" + // " ADD PARTITION (\"col3\" = 'tajo' , \"col4\" = '2015-09-01')"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 400c3915fb..3b22662b16 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -1286,7 +1286,8 @@ public final void testAlterTableRepairPartiton() throws TajoException { "ALTER TABLE partitioned_table DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' )", //1 }; - @Test + // TODO: This should be added at TAJO-1891 +// @Test public final void testAddPartitionAndDropPartition() throws TajoException { String tableName = CatalogUtil.normalizeIdentifier("partitioned_table"); String qualifiedTableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName); @@ -1329,33 +1330,8 @@ public final void testAddPartitionAndDropPartition() throws TajoException { QueryContext qc = new QueryContext(util.getConfiguration(), session); - // Disable the alter table add partition statement temporarily at TAJO-1887 // Testing alter table add partition -// Expr expr = sqlAnalyzer.parse(ALTER_PARTITIONS[0]); -// LogicalPlan rootNode = planner.createPlan(qc, expr); -// LogicalNode plan = rootNode.getRootBlock().getRoot(); -// testJsonSerDerObject(plan); -// assertEquals(NodeType.ROOT, plan.getType()); -// LogicalRootNode root = (LogicalRootNode) plan; -// assertEquals(NodeType.ALTER_TABLE, root.getChild().getType()); -// -// AlterTableNode alterTableNode = root.getChild(); -// -// assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); -// -// assertEquals(alterTableNode.getPartitionColumns().length, 2); -// assertEquals(alterTableNode.getPartitionValues().length, 2); -// -// assertEquals(alterTableNode.getPartitionColumns()[0], "col1"); -// assertEquals(alterTableNode.getPartitionColumns()[1], "col2"); -// -// assertEquals(alterTableNode.getPartitionValues()[0], "1"); -// assertEquals(alterTableNode.getPartitionValues()[1], "2"); -// -// assertEquals(alterTableNode.getLocation(), "hdfs://xxx.com/warehouse/partitioned_table/col1=1/col2=2"); - - // Testing alter table drop partition - Expr expr = sqlAnalyzer.parse(ALTER_PARTITIONS[1]); + Expr expr = sqlAnalyzer.parse(ALTER_PARTITIONS[0]); LogicalPlan rootNode = planner.createPlan(qc, expr); LogicalNode plan = rootNode.getRootBlock().getRoot(); testJsonSerDerObject(plan); @@ -1365,6 +1341,30 @@ public final void testAddPartitionAndDropPartition() throws TajoException { AlterTableNode alterTableNode = root.getChild(); + assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); + + assertEquals(alterTableNode.getPartitionColumns().length, 2); + assertEquals(alterTableNode.getPartitionValues().length, 2); + + assertEquals(alterTableNode.getPartitionColumns()[0], "col1"); + assertEquals(alterTableNode.getPartitionColumns()[1], "col2"); + + assertEquals(alterTableNode.getPartitionValues()[0], "1"); + assertEquals(alterTableNode.getPartitionValues()[1], "2"); + + assertEquals(alterTableNode.getLocation(), "hdfs://xxx.com/warehouse/partitioned_table/col1=1/col2=2"); + + // Testing alter table drop partition + expr = sqlAnalyzer.parse(ALTER_PARTITIONS[1]); + rootNode = planner.createPlan(qc, expr); + plan = rootNode.getRootBlock().getRoot(); + testJsonSerDerObject(plan); + assertEquals(NodeType.ROOT, plan.getType()); + root = (LogicalRootNode) plan; + assertEquals(NodeType.ALTER_TABLE, root.getChild().getType()); + + alterTableNode = root.getChild(); + assertEquals(alterTableNode.getAlterTableOpType(), AlterTableOpType.DROP_PARTITION); assertEquals(alterTableNode.getPartitionColumns().length, 3); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 25ce4f3633..e6f2752b11 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -29,7 +29,6 @@ import org.apache.tajo.exception.UndefinedPartitionException; import org.apache.tajo.exception.UndefinedPartitionMethodException; import org.apache.tajo.exception.UndefinedTableException; -import org.apache.tajo.util.TUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -59,7 +58,7 @@ public final void testAlterTableColumnName() throws Exception { public final void testAlterTableAddNewColumn() throws Exception { List createdNames = executeDDL("table1_ddl.sql", "table1.tbl", "EFG"); executeDDL("alter_table_add_new_column_ddl.sql", null); - assertColumnExists(createdNames.get(0), "cool"); + assertColumnExists(createdNames.get(0),"cool"); } @Test @@ -76,7 +75,8 @@ public final void testAlterTableSetProperty() throws Exception { cleanupQuery(after_res); } - @Test + // TODO: This should be added at TAJO-1891 +// @Test public final void testAlterTableAddPartition() throws Exception { executeDDL("create_partitioned_table.sql", null); @@ -90,12 +90,8 @@ public final void testAlterTableAddPartition() throws Exception { assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(0).getSimpleName(), "col3"); assertEquals(retrieved.getPartitionMethod().getExpressionSchema().getColumn(1).getSimpleName(), "col4"); - // Disable the alter table add partition statement temporarily at TAJO-1887 -// executeDDL("alter_table_add_partition1.sql", null); -// executeDDL("alter_table_add_partition2.sql", null); - List targetPartitions = TUtil.newList(); - targetPartitions.add(CatalogUtil.buildPartitionDescProto("col3=1/col4=2", retrieved.getUri().toString())); - catalog.addPartitions("TestAlterTable", "partitioned_table", targetPartitions, true); + executeDDL("alter_table_add_partition1.sql", null); + executeDDL("alter_table_add_partition2.sql", null); List partitions = catalog.getPartitionsOfTable("TestAlterTable", "partitioned_table"); assertNotNull(partitions); @@ -107,11 +103,10 @@ public final void testAlterTableAddPartition() throws Exception { assertEquals(partitions.get(0).getPartitionKeysList().get(1).getPartitionValue(), "2"); assertNotNull(partitions.get(0).getPath()); - // Disable the alter table add partition statement temporarily at TAJO-1887 -// Path partitionPath = new Path(partitions.get(0).getPath()); -// FileSystem fs = partitionPath.getFileSystem(conf); -// assertTrue(fs.exists(partitionPath)); -// assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); + Path partitionPath = new Path(partitions.get(0).getPath()); + FileSystem fs = partitionPath.getFileSystem(conf); + assertTrue(fs.exists(partitionPath)); + assertTrue(partitionPath.toString().indexOf("col3=1/col4=2") > 0); executeDDL("alter_table_drop_partition1.sql", null); executeDDL("alter_table_drop_partition2.sql", null); @@ -119,8 +114,7 @@ public final void testAlterTableAddPartition() throws Exception { partitions = catalog.getPartitionsOfTable("TestAlterTable", "partitioned_table"); assertNotNull(partitions); assertEquals(partitions.size(), 0); - // Disable the alter table add partition statement temporarily at TAJO-1887 -// assertFalse(fs.exists(partitionPath)); + assertFalse(fs.exists(partitionPath)); catalog.dropTable(tableName); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java b/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java index 3a577e824f..e2968a8664 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/parser/sql/TestSQLAnalyzer.java @@ -75,7 +75,12 @@ public Collection getResourceFiles(String subdir) throws URISyntaxExceptio new Predicate() { @Override public boolean apply(@Nullable FileStatus input) { - return input.isFile(); + // TODO: This should be removed at TAJO-1891 + if (input.getPath().getName().indexOf("add_partition") > -1) { + return false; + } else { + return input.isFile(); + } } } ); diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_1.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_1.sql new file mode 100644 index 0000000000..0f91b68dd7 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_1.sql @@ -0,0 +1 @@ +ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_2.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_2.sql new file mode 100644 index 0000000000..ee4df750fe --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_2.sql @@ -0,0 +1 @@ +ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx.com/warehouse/table1/col1=1/col2=2' \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_3.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_3.sql new file mode 100644 index 0000000000..79d14266ce --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_3.sql @@ -0,0 +1,2 @@ +ALTER TABLE table1 ADD PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) +LOCATION 'hdfs://xxx.com/warehouse/table1/col1=2015/col2=01/col3=11' \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_4.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_4.sql new file mode 100644 index 0000000000..bc22a1418f --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_4.sql @@ -0,0 +1 @@ +ALTER TABLE table1 ADD PARTITION (col1 = 'TAJO' ) \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_5.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_5.sql new file mode 100644 index 0000000000..127d9993c3 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/alter_table_add_partition_5.sql @@ -0,0 +1 @@ +ALTER TABLE table1 ADD IF NOT EXISTS PARTITION (col1 = 'TAJO' ) \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 207fd0efde..57d01aa4a0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -477,55 +477,54 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer .SET_PROPERTY)); break; case ADD_PARTITION: -// pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), alterTable.getPartitionValues()); -// ensureColumnPartitionKeys(qualifiedName, alterTable.getPartitionColumns()); -// -// // checking a duplicated partition -// boolean duplicatedPartition = true; -// try { -// catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); -// } catch (UndefinedPartitionException e) { -// duplicatedPartition = false; -// } -// -// if (duplicatedPartition && !alterTable.isIfNotExists()) { -// throw new DuplicatePartitionException(pair.getSecond()); -// } else if (!duplicatedPartition) { -// if (alterTable.getLocation() != null) { -// partitionPath = new Path(alterTable.getLocation()); -// } else { -// // If location is not specified, the partition's location will be set using the table location. -// partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); -// alterTable.setLocation(partitionPath.toString()); -// } -// -// FileSystem fs = partitionPath.getFileSystem(context.getConf()); -// -// // If there is a directory which was assumed to be a partitioned directory and users don't input another -// // location, this will throw exception. -// Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond()); -// -// if (fs.exists(assumedDirectory) && !assumedDirectory.equals(partitionPath)) { -// throw new AmbiguousPartitionDirectoryExistException(assumedDirectory.toString()); -// } -// -// long numBytes = 0L; -// if (fs.exists(partitionPath)) { -// ContentSummary summary = fs.getContentSummary(partitionPath); -// numBytes = summary.getLength(); -// } -// -// catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), -// alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION, numBytes)); -// -// // If the partition's path doesn't exist, this would make the directory by force. -// if (!fs.exists(partitionPath)) { -// fs.mkdirs(partitionPath); -// } -// } -// -// break; - throw new TajoRuntimeException(new UnsupportedException("ADD PARTITION")); + pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), alterTable.getPartitionValues()); + ensureColumnPartitionKeys(qualifiedName, alterTable.getPartitionColumns()); + + // checking a duplicated partition + boolean duplicatedPartition = true; + try { + catalog.getPartition(databaseName, simpleTableName, pair.getSecond()); + } catch (UndefinedPartitionException e) { + duplicatedPartition = false; + } + + if (duplicatedPartition && !alterTable.isIfNotExists()) { + throw new DuplicatePartitionException(pair.getSecond()); + } else if (!duplicatedPartition) { + if (alterTable.getLocation() != null) { + partitionPath = new Path(alterTable.getLocation()); + } else { + // If location is not specified, the partition's location will be set using the table location. + partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); + alterTable.setLocation(partitionPath.toString()); + } + + FileSystem fs = partitionPath.getFileSystem(context.getConf()); + + // If there is a directory which was assumed to be a partitioned directory and users don't input another + // location, this will throw exception. + Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond()); + + if (fs.exists(assumedDirectory) && !assumedDirectory.equals(partitionPath)) { + throw new AmbiguousPartitionDirectoryExistException(assumedDirectory.toString()); + } + + long numBytes = 0L; + if (fs.exists(partitionPath)) { + ContentSummary summary = fs.getContentSummary(partitionPath); + numBytes = summary.getLength(); + } + + catalog.alterTable(CatalogUtil.addOrDropPartition(qualifiedName, alterTable.getPartitionColumns(), + alterTable.getPartitionValues(), alterTable.getLocation(), AlterTableType.ADD_PARTITION, numBytes)); + + // If the partition's path doesn't exist, this would make the directory by force. + if (!fs.exists(partitionPath)) { + fs.mkdirs(partitionPath); + } + } + + break; case DROP_PARTITION: ensureColumnPartitionKeys(qualifiedName, alterTable.getPartitionColumns()); pair = CatalogUtil.getPartitionKeyNamePair(alterTable.getPartitionColumns(), alterTable.getPartitionValues()); diff --git a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java index 930c959458..717366a88b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java @@ -1910,14 +1910,12 @@ public Expr visitAlter_table_statement(Alter_table_statementContext ctx) { } alterTable.setColumns(columns); alterTable.setValues(values); - // Disable the alter table add partition statement temporarily at TAJO-1887 -// if (ctx.LOCATION() != null) { -// String path = stripQuote(ctx.path.getText()); -// alterTable.setLocation(path); -// } -// alterTable.setIfNotExists(checkIfExist(ctx.if_not_exists())); - + if (ctx.LOCATION() != null) { + String path = stripQuote(ctx.path.getText()); + alterTable.setLocation(path); + } alterTable.setPurge(checkIfExist(ctx.PURGE())); + alterTable.setIfNotExists(checkIfExist(ctx.if_not_exists())); alterTable.setIfExists(checkIfExist(ctx.if_exists())); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 2c7feaca7a..821df70347 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -354,4 +354,14 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws return expr; } + + // TODO: This should be removed at TAJO-1891 + @Override + public Expr visitAlterTable(Context context, Stack stack, AlterTable expr) throws TajoException { + if (expr.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION) { + context.state.addVerification(new UnsupportedException("ADD PARTITION")); + } + + return expr; + } } diff --git a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 index 630f46bbd7..e2693eae44 100644 --- a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 +++ b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 @@ -1622,6 +1622,7 @@ alter_table_statement : ALTER TABLE table_name RENAME TO table_name | ALTER TABLE table_name RENAME COLUMN column_name TO column_name | ALTER TABLE table_name ADD COLUMN field_element + | ALTER TABLE table_name ADD (if_not_exists)? PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (LOCATION path=Character_String_Literal)? | ALTER TABLE table_name DROP (if_exists)? PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (PURGE)? | ALTER TABLE table_name SET PROPERTY property_list | ALTER TABLE table_name REPAIR PARTITION From 566e56d646081b0cb5a24c1baf3132cdf4aa66e1 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 11:36:09 +0900 Subject: [PATCH 08/16] Recover CatalogUtil --- .../org/apache/tajo/catalog/CatalogUtil.java | 48 +------------------ 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 3eeae125da..996bfaf733 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -33,10 +33,10 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableIdentifierProto; -import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; +import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UndefinedOperatorException; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; @@ -44,7 +44,6 @@ import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; -import java.io.File; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -1044,49 +1043,4 @@ public int compare(String o1, String o2) { return originlSchema.getColumnId(o1) - originlSchema.getColumnId(o2); } } - - public static PartitionDesc buildPartitionDesc(String partitionName, String tablePath) { - PartitionDesc partitionDesc = new PartitionDesc(); - partitionDesc.setPartitionName(partitionName); - - String[] partitionNames = partitionName.split("/"); - partitionDesc.setPartitionKeys(buildPartitionKeyProtos(partitionNames)); - partitionDesc.setPath(tablePath + File.separator + partitionName); - return partitionDesc; - } - - public static PartitionDescProto buildPartitionDescProto(String partitionName, String tablePath) { - PartitionDescProto.Builder partitionDescProto = PartitionDescProto.newBuilder(); - partitionDescProto.setPartitionName(partitionName); - - String[] partitionNames = partitionName.split("/"); - partitionDescProto.addAllPartitionKeys(buildPartitionKeyProtos(partitionNames)); - partitionDescProto.setPath(tablePath + File.separator + partitionName); - return partitionDescProto.build(); - } - - private static List buildPartitionKeyProtos(String[] partitionNames) { - List partitionKeyList = new ArrayList<>(); - for(int i = 0; i < partitionNames.length; i++) { - String [] splits = partitionNames[i].split("="); - String columnName = "", partitionValue = ""; - if (splits.length == 2) { - columnName = splits[0]; - partitionValue = splits[1]; - } else if (splits.length == 1) { - if (partitionNames[i].charAt(0) == '=') { - partitionValue = splits[0]; - } else { - columnName = ""; - } - } - - PartitionKeyProto.Builder builder = PartitionKeyProto.newBuilder(); - builder.setColumnName(columnName); - builder.setPartitionValue(partitionValue); - partitionKeyList.add(builder.build()); - } - - return partitionKeyList; - } } From 16bd54e0cf7e4fb13ebf50f6ec5b2de8cd3df635 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 11:38:28 +0900 Subject: [PATCH 09/16] Update comments --- .../main/java/org/apache/tajo/cli/tools/TajoDump.java | 3 ++- .../java/org/apache/tajo/cli/tools/TestTajoDump.java | 10 +++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index 0ef98a5b79..42e28aa671 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -192,13 +192,14 @@ private static void dumpDatabase(TajoClient client, String databaseName, PrintWr writer.write("--\n"); writer.write(String.format("-- Table Partitions: %s%n", tableName)); writer.write("--\n"); - // Disable the alter table add partition statement temporarily at TAJO-1887 + // TODO: This should be improved at TAJO-1891 // List partitionProtos = client.getPartitionsOfTable(fqName); // for (PartitionDescProto eachPartitionProto : partitionProtos) { // writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto)); // } writer.write(String.format("ALTER TABLE %s REPAIR PARTITION%n", CatalogUtil.denormalizeIdentifier(databaseName) + "." + CatalogUtil.denormalizeIdentifier(tableName))); + writer.write("\n\n"); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java index 0ae9b3e39a..13936e6853 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -113,20 +113,16 @@ public void testPartitionsDump() throws Exception { + " partition by column(\"col3\" int4, \"col4\" int4)" ); - // TODO: This should be added at TAJO-1891 + // TODO: This should be improved at TAJO-1891 // executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName3\"" + // " ADD PARTITION (\"col3\" = 1 , \"col4\" = 2)"); - - // TODO: This should be removed at TAJO-1891 +// executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"" + +// " ADD PARTITION (\"col3\" = 'tajo' , \"col4\" = '2015-09-01')"); executeString("create table \"" + getCurrentDatabase() + "\".\"TableName4\"" + " (\"col1\" int4, \"col2\" int4) " + " partition by column(\"col3\" TEXT, \"col4\" date)" ); - // TODO: This should be added at TAJO-1891 -// executeString("ALTER TABLE \"" + getCurrentDatabase() + "\".\"TableName4\"" + -// " ADD PARTITION (\"col3\" = 'tajo' , \"col4\" = '2015-09-01')"); - try { UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); ByteArrayOutputStream bos = new ByteArrayOutputStream(); From 487a796569be62aa3c02648fa9cc5b5da98fb5c0 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 11:40:50 +0900 Subject: [PATCH 10/16] Remove unnecessary comments --- .../org/apache/tajo/catalog/store/TestHiveCatalogStore.java | 1 - .../apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java | 1 - .../java/org/apache/tajo/catalog/TestCatalogExceptions.java | 3 --- .../org/apache/tajo/engine/planner/TestLogicalPlanner.java | 1 - .../test/java/org/apache/tajo/engine/query/TestAlterTable.java | 1 - 5 files changed, 7 deletions(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index 9854e951ed..3c2c6c1586 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -220,7 +220,6 @@ public void testTableWithNullValue() throws Exception { } // TODO: This should be added at TAJO-1891 -// @Test public void testAddTableByPartition() throws Exception { TableMeta meta = new TableMeta("TEXT", new KeyValueSet()); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index 24c0abfa94..71c3d6a2b8 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -146,7 +146,6 @@ public void testTable() throws Exception { } // TODO: This should be added at TAJO-1891 -// @Test public void testTablePartition() throws Exception { ////////////////////////////////////////////////////////////////////////////// // Test add partition diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java index e27f0c067a..a01e0a49d7 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java @@ -158,7 +158,6 @@ public void testUpdateTableStatsOfUndefinedTable() throws Exception { } // TODO: This should be added at TAJO-1891 -// @Test public void testAddPartitionWithWrongUri() throws Exception { // TODO: currently, wrong uri does not occur any exception. String partitionName = "DaTe=/=AaA"; @@ -173,7 +172,6 @@ public void testAddPartitionWithWrongUri() throws Exception { } // TODO: This should be added at TAJO-1891 -// @Test(expected = DuplicatePartitionException.class) public void testAddDuplicatePartition() throws Exception { String partitionName = "DaTe=bBb/dAtE=AaA"; PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); @@ -197,7 +195,6 @@ public void testAddDuplicatePartition() throws Exception { } // TODO: This should be added at TAJO-1891 -// @Test(expected = UndefinedTableException.class) public void testAddPartitionToUndefinedTable() throws Exception { String partitionName = "DaTe=bBb/dAtE=AaA"; PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 3b22662b16..3e991434e4 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -1287,7 +1287,6 @@ public final void testAlterTableRepairPartiton() throws TajoException { }; // TODO: This should be added at TAJO-1891 -// @Test public final void testAddPartitionAndDropPartition() throws TajoException { String tableName = CatalogUtil.normalizeIdentifier("partitioned_table"); String qualifiedTableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index e6f2752b11..9a30012a2a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -76,7 +76,6 @@ public final void testAlterTableSetProperty() throws Exception { } // TODO: This should be added at TAJO-1891 -// @Test public final void testAlterTableAddPartition() throws Exception { executeDDL("create_partitioned_table.sql", null); From 77219c0f03132fe517f49b1537b7549866ea43f4 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 11:43:04 +0900 Subject: [PATCH 11/16] Remove unnecessary comments --- .../src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java | 5 +---- .../src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java | 6 +----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index a10e290f8f..d6bdc1f1cf 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -473,9 +473,7 @@ public void testNonForwardQueryPause() throws Exception { } } - // Disable the alter table add partition statement temporarily at TAJO-1887 -/* - @Test + // TODO: This should be added at TAJO-1891 public void testAlterTableAddDropPartition() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition"); @@ -505,7 +503,6 @@ public void testAlterTableAddDropPartition() throws Exception { String consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); } -*/ public static class TajoCliOutputTestFormatter extends DefaultTajoCliOutputFormatter { @Override diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index 209a82dad8..db0aff588f 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -589,10 +589,7 @@ public void testSortWithDateTime() throws Exception { } } - - // Disable the alter table add partition statement temporarily at TAJO-1887 - /* - @Test + // TODO: This should be added at TAJO-1891 public void testAlterTableAddPartition() throws Exception { Statement stmt = null; ResultSet resultSet = null; @@ -631,7 +628,6 @@ public void testAlterTableAddPartition() throws Exception { } } } - */ @Test public void testMaxRows() throws Exception { From 0d82d32b25cff8941dc9a5a3012f57b7879a3d94 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 12:00:44 +0900 Subject: [PATCH 12/16] Remove unnecessary character --- tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java | 2 +- .../resources/results/TestTajoDump/testPartitionsDump.result | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index 42e28aa671..aeae8454f0 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -197,7 +197,7 @@ private static void dumpDatabase(TajoClient client, String databaseName, PrintWr // for (PartitionDescProto eachPartitionProto : partitionProtos) { // writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto)); // } - writer.write(String.format("ALTER TABLE %s REPAIR PARTITION%n", + writer.write(String.format("ALTER TABLE %s REPAIR PARTITION", CatalogUtil.denormalizeIdentifier(databaseName) + "." + CatalogUtil.denormalizeIdentifier(tableName))); writer.write("\n\n"); diff --git a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result index 73db854bff..d678f8919a 100644 --- a/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result +++ b/tajo-core-tests/src/test/resources/results/TestTajoDump/testPartitionsDump.result @@ -21,7 +21,6 @@ ALTER TABLE "TestTajoDump"."TableName3" REPAIR PARTITION - -- -- Name: "TestTajoDump"."TableName4"; Type: TABLE; Storage: TEXT -- From 41504f3e9edb804e04c18f936c55ea314ed83fcc Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 12:09:01 +0900 Subject: [PATCH 13/16] Use NotImplementedException instead of UnsupportedException --- .../org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 821df70347..00ff019c94 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -359,7 +359,7 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws @Override public Expr visitAlterTable(Context context, Stack stack, AlterTable expr) throws TajoException { if (expr.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION) { - context.state.addVerification(new UnsupportedException("ADD PARTITION")); + context.state.addVerification(new NotImplementedException("ADD PARTITION")); } return expr; From 809de5f0afc305f33098624a2e47b1802f1f92f9 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 12:13:19 +0900 Subject: [PATCH 14/16] Use NotImplementedException instead of UnsupportedExceptio --- .../src/main/java/org/apache/tajo/catalog/CatalogServer.java | 2 +- .../main/java/org/apache/tajo/exception/ReturnStateUtil.java | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index 880687dbc4..9bc85c7a4f 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -461,7 +461,7 @@ public ReturnState alterTable(RpcController controller, AlterTableDescProto prot // TODO: This should be removed at TAJO-1891 if (proto.getAlterTableType() == CatalogProtos.AlterTableType.ADD_PARTITION) { - return errFeatureNotSupported("ADD PARTTIION"); + return errFeatureNotImplemented("ADD PARTTIION"); } wlock.lock(); diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java index cf98f37847..3257f46b2a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ReturnStateUtil.java @@ -201,4 +201,9 @@ public static ReturnState errDuplicateIndex(String indexName) { public static ReturnState errDuplicateFunction(String signature) { return returnError(ResultCode.DUPLICATE_FUNCTION, signature); } + + public static ReturnState errFeatureNotImplemented(String feature) { + return returnError(ResultCode.NOT_IMPLEMENTED, feature); + } + } From 059f01866b57a6ecd3d3fa4b6a8ebb21daa12b96 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 13:44:04 +0900 Subject: [PATCH 15/16] Add negative unit test cases --- .../tajo/catalog/AbstractCatalogClient.java | 3 +- .../apache/tajo/catalog/CatalogService.java | 10 +++- .../tajo/catalog/TestCatalogExceptions.java | 50 ++++++++++++------- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 12 +++++ ...AddPartitionNotimplementedException.result | 2 + 5 files changed, 57 insertions(+), 20 deletions(-) create mode 100644 tajo-core-tests/src/test/resources/results/TestTajoCli/testAddPartitionNotimplementedException.result diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 38e02ae906..be6fe651df 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -921,7 +921,7 @@ public final boolean containFunction(final String signature, FunctionType funcTy public final void alterTable(final AlterTableDesc desc) throws DuplicateDatabaseException, DuplicateTableException, DuplicateColumnException, DuplicatePartitionException, UndefinedDatabaseException, UndefinedTableException, UndefinedColumnException, UndefinedPartitionMethodException, - InsufficientPrivilegeException, UndefinedPartitionException { + InsufficientPrivilegeException, UndefinedPartitionException, NotImplementedException { try { final BlockingInterface stub = getStub(); @@ -936,6 +936,7 @@ public final void alterTable(final AlterTableDesc desc) throws DuplicateDatabase throwsIfThisError(state, UndefinedPartitionException.class); throwsIfThisError(state, UndefinedPartitionMethodException.class); throwsIfThisError(state, InsufficientPrivilegeException.class); + throwsIfThisError(state, NotImplementedException.class); ensureOk(state); } catch (ServiceException e) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java index 2140b4980e..54d22e7878 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java @@ -228,11 +228,18 @@ FunctionDesc getFunction(String signature, FunctionType funcType, DataType... pa /** * Add a table via table description * + * @param desc * @throws DuplicateColumnException * @throws DuplicateTableException * @throws InsufficientPrivilegeException * @throws UndefinedColumnException * @throws UndefinedTableException + * @throws DuplicateDatabaseException + * @throws DuplicatePartitionException + * @throws UndefinedDatabaseException + * @throws UndefinedPartitionMethodException + * @throws UndefinedPartitionException + * @throws NotImplementedException * * @see AlterTableDesc */ @@ -246,7 +253,8 @@ void alterTable(AlterTableDesc desc) DuplicatePartitionException, UndefinedDatabaseException, UndefinedPartitionMethodException, - UndefinedPartitionException; + UndefinedPartitionException, + NotImplementedException; void updateTableStats(UpdateTableStatsProto stats) throws UndefinedTableException, InsufficientPrivilegeException; } diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java index a01e0a49d7..5d4065673b 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java @@ -79,11 +79,11 @@ public void testDropDefaultTablespace() throws Exception { @Test(expected = TajoInternalError.class) public void testAlterTablespaceWithWrongUri() throws Exception { catalog.alterTablespace(AlterTablespaceProto.newBuilder(). - setSpaceName("space1"). - addCommand( - AlterTablespaceCommand.newBuilder(). - setType(AlterTablespaceType.LOCATION). - setLocation("hdfs:")).build()); + setSpaceName("space1"). + addCommand( + AlterTablespaceCommand.newBuilder(). + setType(AlterTablespaceType.LOCATION). + setLocation("hdfs:")).build()); } @Test(expected = UndefinedTablespaceException.class) @@ -121,11 +121,11 @@ public void testCreateTableWithWrongUri() throws Exception { schema.addColumn(CatalogUtil.buildFQName(tableName, "cOlumn"), Type.INT8); Path path = new Path(CommonTestingUtil.getTestDir(), tableName); catalog.createTable( - new TableDesc( - CatalogUtil.buildFQName("TestDatabase1", tableName), - schema, - new TableMeta("TEXT", new KeyValueSet()), - path.toUri(), true)); + new TableDesc( + CatalogUtil.buildFQName("TestDatabase1", tableName), + schema, + new TableMeta("TEXT", new KeyValueSet()), + path.toUri(), true)); } @Test(expected = DuplicateTableException.class) @@ -147,14 +147,14 @@ public void dropUndefinedTable() throws Exception { @Test(expected = UndefinedTableException.class) public void testUpdateTableStatsOfUndefinedTable() throws Exception { catalog.updateTableStats( - UpdateTableStatsProto.newBuilder(). - setTableName(CatalogUtil.buildFQName("TestDatabase1", "undefined")). - setStats( - TableStatsProto.newBuilder(). - setNumRows(0). - setNumBytes(0). - build()). - build()); + UpdateTableStatsProto.newBuilder(). + setTableName(CatalogUtil.buildFQName("TestDatabase1", "undefined")). + setStats( + TableStatsProto.newBuilder(). + setNumRows(0). + setNumBytes(0). + build()). + build()); } // TODO: This should be added at TAJO-1891 @@ -207,6 +207,20 @@ public void testAddPartitionToUndefinedTable() throws Exception { catalog.alterTable(alterTableDesc); } + // TODO: This should be removed at TAJO-1891 + @Test(expected = NotImplementedException.class) + public void testAddPartitionNotimplementedException() throws Exception { + String partitionName = "DaTe=/=AaA"; + PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); + + AlterTableDesc alterTableDesc = new AlterTableDesc(); + alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setPartitionDesc(partitionDesc); + alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); + + catalog.alterTable(alterTableDesc); + } + @Test(expected = UndefinedPartitionException.class) public void testDropUndefinedPartition() throws Exception { String partitionName = "DaTe=undefined/dAtE=undefined"; diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index d6bdc1f1cf..f265e55638 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -473,6 +473,18 @@ public void testNonForwardQueryPause() throws Exception { } } + // TODO: This should be removed at TAJO-1891 + @Test + public void testAddPartitionNotimplementedException() throws Exception { + String tableName = CatalogUtil.normalizeIdentifier("testAddPartitionNotimplementedException"); + tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); + tajoCli.executeScript("alter table " + tableName + " add partition (key2 = 0.1)"); + + String consoleResult; + consoleResult = new String(out.toByteArray()); + assertOutputResult(consoleResult); + } + // TODO: This should be added at TAJO-1891 public void testAlterTableAddDropPartition() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition"); diff --git a/tajo-core-tests/src/test/resources/results/TestTajoCli/testAddPartitionNotimplementedException.result b/tajo-core-tests/src/test/resources/results/TestTajoCli/testAddPartitionNotimplementedException.result new file mode 100644 index 0000000000..4aab8c8045 --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestTajoCli/testAddPartitionNotimplementedException.result @@ -0,0 +1,2 @@ +OK +ERROR: not implemented feature: ADD PARTITION \ No newline at end of file From adb02a5577c9dd0355f6833ae7ac7eb9977d6263 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 25 Sep 2015 19:35:26 +0900 Subject: [PATCH 16/16] Remove description about add partition --- .../src/main/sphinx/sql_language/alter_table.rst | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst index 959ebcc5c6..47e74922b9 100644 --- a/tajo-docs/src/main/sphinx/sql_language/alter_table.rst +++ b/tajo-docs/src/main/sphinx/sql_language/alter_table.rst @@ -65,22 +65,6 @@ SET PROPERTY This statement will allow users to change a table property. -======================== -ADD PARTITION -======================== - -*Synopsis* - -.. code-block:: sql - - ALTER TABLE [IF NOT EXISTS] ADD PARTITION ( = , ...) [LOCATION = ] - - For example: - ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) - ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx.com/warehouse/table1/col1=1/col2=2' - -You can use ``ALTER TABLE ADD PARTITION`` to add partitions to a table. The location must be a directory inside of which data files reside. If the location doesn't exist on the file system, Tajo will make the location by force. ``ADD PARTITION`` changes the table metadata, but does not load data. If the data does not exist in the partition's location, queries will not return any results. An error is thrown if the partition for the table already exists. You can use ``IF NOT EXISTS`` to skip the error. - ======================== DROP PARTITION ========================