From ead3c99954ba417852085221bd3b2a2680893be5 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 7 Jan 2015 16:14:34 +0900 Subject: [PATCH 1/4] TAJO-1260: Add ALTER TABLE ADD/DROP PARTITION statement to parser --- .../org/apache/tajo/algebra/AlterTable.java | 34 ++++++++++- .../apache/tajo/algebra/AlterTableOpType.java | 2 +- .../apache/tajo/engine/parser/SQLParser.g4 | 10 ++++ .../tajo/engine/parser/SQLAnalyzer.java | 56 ++++++++++++++---- .../tajo/engine/parser/TestSQLAnalyzer.java | 58 +++++++++++++++++-- .../default/alter_table_add_partition_1.sql | 1 + .../default/alter_table_add_partition_2.sql | 1 + .../default/alter_table_drop_partition_1.sql | 1 + 8 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/default/alter_table_add_partition_1.sql create mode 100644 tajo-core/src/test/resources/queries/default/alter_table_add_partition_2.sql create mode 100644 tajo-core/src/test/resources/queries/default/alter_table_drop_partition_1.sql diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java index 6d72472784..dc68fc1b7b 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java @@ -39,6 +39,13 @@ public class AlterTable extends Expr { @Expose @SerializedName("AlterTableType") private AlterTableOpType alterTableOpType; + @Expose @SerializedName("Columns") + ColumnReferenceExpr [] columns; + @Expose @SerializedName("Values") + private Expr[] values; + @Expose @SerializedName("location") + private String location; + public AlterTable(final String tableName) { super(OpType.AlterTable); this.tableName = tableName; @@ -93,6 +100,18 @@ public void setAlterTableOpType(AlterTableOpType alterTableOpType) { this.alterTableOpType = alterTableOpType; } + public ColumnReferenceExpr[] getColumns() { return columns; } + + public void setColumns(ColumnReferenceExpr[] columns) { this.columns = columns; } + + public Expr[] getValues() { return values; } + + public void setValues(Expr[] values) { this.values = values; } + + public String getLocation() { return location; } + + public void setLocation(String location) { this.location = location; } + @Override public int hashCode() { return Objects.hashCode(tableName, @@ -100,7 +119,11 @@ public int hashCode() { null != columnName ? Objects.hashCode(columnName) : columnName, null != newColumnName ? Objects.hashCode(newColumnName) : newColumnName, null != addNewColumn ? Objects.hashCode(addNewColumn) : addNewColumn, - null != alterTableOpType ? Objects.hashCode(alterTableOpType) : alterTableOpType); + null != alterTableOpType ? Objects.hashCode(alterTableOpType) : alterTableOpType, + null != columns ? Objects.hashCode(columns) : columns, + null != values ? Objects.hashCode(values) : values, + null != location ? Objects.hashCode(location) : location + ); } @@ -112,7 +135,11 @@ boolean equalsTo(Expr expr) { TUtil.checkEquals(columnName, another.columnName) && TUtil.checkEquals(newColumnName, another.newColumnName) && TUtil.checkEquals(addNewColumn, another.addNewColumn) && - TUtil.checkEquals(alterTableOpType, another.alterTableOpType); + TUtil.checkEquals(alterTableOpType, another.alterTableOpType) && + TUtil.checkEquals(columns, another.columns) && + TUtil.checkEquals(values, another.values) && + TUtil.checkEquals(location, another.location) + ; } @Override @@ -124,6 +151,9 @@ public Object clone() throws CloneNotSupportedException { alter.newColumnName = newColumnName; alter.addNewColumn = (ColumnDefinition) addNewColumn.clone(); alter.alterTableOpType = alterTableOpType; + alter.columns = columns; + alter.values = values; + alter.location = location; return alter; } } diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java index 67b28a223e..5d4f38196f 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTableOpType.java @@ -18,5 +18,5 @@ package org.apache.tajo.algebra; public enum AlterTableOpType { - RENAME_TABLE, RENAME_COLUMN, ADD_COLUMN + RENAME_TABLE, RENAME_COLUMN, ADD_COLUMN, ADD_PARTITION, DROP_PARTITION } diff --git a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 0bc89db69f..5e453167a7 100644 --- a/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -1593,4 +1593,14 @@ alter_table_statement : ALTER TABLE table_name RENAME TO table_name | ALTER TABLE table_name RENAME COLUMN column_name TO column_name | ALTER TABLE table_name ADD COLUMN field_element + | ALTER TABLE table_name (if_not_exists)? ADD PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN (LOCATION path=Character_String_Literal)? + | ALTER TABLE table_name (if_exists)? DROP PARTITION LEFT_PAREN partition_column_value_list RIGHT_PAREN + ; + +partition_column_value_list + : partition_column_value (COMMA partition_column_value)* + ; + +partition_column_value + : identifier EQUAL row_value_predicand ; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 3669625117..375fe0edaa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1734,6 +1734,24 @@ public Expr visitAlter_table_statement(SQLParser.Alter_table_statementContext ct alterTable.setAddNewColumn(columnDefinition); } + if (checkIfExist(ctx.partition_column_value_list())) { + List columnValueList = ctx.partition_column_value_list().partition_column_value(); + int size = columnValueList.size(); + ColumnReferenceExpr[] columns = new ColumnReferenceExpr[size]; + Expr[] values = new Expr[size]; + for (int i = 0; i < size; i++) { + Partition_column_valueContext columnValue = columnValueList.get(i); + columns[i] = new ColumnReferenceExpr(columnValue.identifier().getText()); + values[i] = visitRow_value_predicand(columnValue.row_value_predicand()); + } + alterTable.setColumns(columns); + alterTable.setValues(values); + if (ctx.LOCATION() != null) { + String path = stripQuote(ctx.path.getText()); + alterTable.setLocation(path); + } + } + alterTable.setAlterTableOpType(determineAlterTableType(ctx)); return alterTable; @@ -1745,23 +1763,35 @@ private AlterTableOpType determineAlterTableType(SQLParser.Alter_table_statement final int COLUMN_MASK = 00000010; final int TO_MASK = 00000100; final int ADD_MASK = 00001000; + final int DROP_MASK = 00001001; + final int PARTITION_MASK = 00000020; int val = 00000000; for (int idx = 1; idx < ctx.getChildCount(); idx++) { if (ctx.getChild(idx) instanceof TerminalNode) { - if (((TerminalNode) ctx.getChild(idx)).getSymbol().getType() == RENAME) { - val = val | RENAME_MASK; - } - if (((TerminalNode) ctx.getChild(idx)).getSymbol().getType() == COLUMN) { - val = val | COLUMN_MASK; - } - if (((TerminalNode) ctx.getChild(idx)).getSymbol().getType() == TO) { - val = val | TO_MASK; - } - if (((TerminalNode) ctx.getChild(idx)).getSymbol().getType() == ADD) { - val = val | ADD_MASK; + switch (((TerminalNode) ctx.getChild(idx)).getSymbol().getType()) { + case RENAME: + val = val | RENAME_MASK; + break; + case COLUMN: + val = val | COLUMN_MASK; + break; + case TO: + val = val | TO_MASK; + break; + case ADD: + val = val | ADD_MASK; + break; + case DROP: + val = val | DROP_MASK; + break; + case PARTITION: + val = val | PARTITION_MASK; + break; + default: + break; } } } @@ -1777,6 +1807,10 @@ private AlterTableOpType evaluateAlterTableOperationTye(final int value) { return AlterTableOpType.RENAME_COLUMN; case 520: return AlterTableOpType.ADD_COLUMN; + case 528: + return AlterTableOpType.ADD_PARTITION; + case 529: + return AlterTableOpType.DROP_PARTITION; default: return null; } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index 272f718f11..c42d6e6c20 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -20,10 +20,7 @@ import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CommonTokenStream; -import org.apache.tajo.algebra.CreateTable; -import org.apache.tajo.algebra.Expr; -import org.apache.tajo.algebra.LiteralValue; -import org.apache.tajo.algebra.OpType; +import org.apache.tajo.algebra.*; import org.apache.tajo.engine.parser.SQLParser.SqlContext; import org.apache.tajo.util.FileUtil; import org.junit.Test; @@ -366,6 +363,59 @@ public void testCreateTablePartitionByColumn() throws IOException { assertEquals("col5", columnPartition.getColumns()[2].getColumnName()); } + @Test + public void testAlterTableAddPartition1() throws IOException { + String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/alter_table_add_partition_1.sql")); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); + assertEquals(2, alterTable.getColumns().length); + assertEquals(2, alterTable.getValues().length); + assertEquals("col1", alterTable.getColumns()[0].getName()); + assertEquals("col2", alterTable.getColumns()[1].getName()); + LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; + assertEquals("1", value1.getValue()); + LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; + assertEquals("2", value2.getValue()); + } + + @Test + public void testAlterTableAddPartition2() throws IOException { + String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/alter_table_add_partition_2.sql")); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); + assertEquals(2, alterTable.getColumns().length); + assertEquals(2, alterTable.getValues().length); + assertEquals("col1", alterTable.getColumns()[0].getName()); + assertEquals("col2", alterTable.getColumns()[1].getName()); + LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; + assertEquals("1", value1.getValue()); + LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; + assertEquals("2", value2.getValue()); + assertEquals(alterTable.getLocation(), "hdfs://xxx.com/warehouse/table1/col1=1/col2=2"); + } + + + @Test + public void testAlterTableDropPartition1() throws IOException { + String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/alter_table_drop_partition_1.sql")); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.DROP_PARTITION); + assertEquals(2, alterTable.getColumns().length); + assertEquals(2, alterTable.getValues().length); + assertEquals("col1", alterTable.getColumns()[0].getName()); + assertEquals("col2", alterTable.getColumns()[1].getName()); + LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; + assertEquals("1", value1.getValue()); + LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; + assertEquals("2", value2.getValue()); + } + @Test public void testTableSubQuery1() throws IOException { String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/table_subquery1.sql")); diff --git a/tajo-core/src/test/resources/queries/default/alter_table_add_partition_1.sql b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_1.sql new file mode 100644 index 0000000000..0f91b68dd7 --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_1.sql @@ -0,0 +1 @@ +ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/default/alter_table_add_partition_2.sql b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_2.sql new file mode 100644 index 0000000000..ee4df750fe --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_2.sql @@ -0,0 +1 @@ +ALTER TABLE table1 ADD PARTITION (col1 = 1 , col2 = 2) LOCATION 'hdfs://xxx.com/warehouse/table1/col1=1/col2=2' \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_1.sql b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_1.sql new file mode 100644 index 0000000000..6e2ad7c799 --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_1.sql @@ -0,0 +1 @@ +ALTER TABLE table1 DROP PARTITION (col1 = 1 , col2 = 2) \ No newline at end of file From 244063d3e369273b00c61524807556ccf7f749b0 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 7 Jan 2015 17:36:58 +0900 Subject: [PATCH 2/4] Use FileUtil.readTextFileFromResource instead of direct paths and added more unit tests. --- .../tajo/engine/parser/TestSQLAnalyzer.java | 184 ++++++++++++------ .../default/alter_table_add_partition_3.sql | 2 + .../default/alter_table_add_partition_4.sql | 1 + .../default/alter_table_drop_partition_2.sql | 1 + .../default/alter_table_drop_partition_3.sql | 1 + 5 files changed, 131 insertions(+), 58 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/default/alter_table_add_partition_3.sql create mode 100644 tajo-core/src/test/resources/queries/default/alter_table_add_partition_4.sql create mode 100644 tajo-core/src/test/resources/queries/default/alter_table_drop_partition_2.sql create mode 100644 tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java index c42d6e6c20..9dfe814ff7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/parser/TestSQLAnalyzer.java @@ -33,7 +33,7 @@ import static org.junit.Assert.assertTrue; /** - * This unit tests uses a number of query files located in tajo/tajo-core/src/test/resources/queries. + * This unit tests uses a number of query files located in tajo/tajo-core/queries. * So, you must set tajo/tajo-core/ as the working directory. */ public class TestSQLAnalyzer { @@ -60,223 +60,223 @@ public void assertParseResult(String sqlFileName, String resultFileName) throws @Test public void testSelect1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/select_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/select_1.sql"); parseQuery(sql); } @Test public void testSelect2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/select_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/select_2.sql"); parseQuery(sql); } @Test public void testSelect3() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/select_3.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/select_3.sql"); parseQuery(sql); } @Test public void testSelect4() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/select_4.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/select_4.sql"); parseQuery(sql); } @Test public void testSelect5() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/select_5.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/select_5.sql"); parseQuery(sql); } @Test public void testAsterisk1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/asterisk_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/asterisk_1.sql"); parseQuery(sql); } @Test public void testAsterisk2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/asterisk_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/asterisk_2.sql"); parseQuery(sql); } @Test public void testAsterisk3() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/asterisk_3.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/asterisk_3.sql"); parseQuery(sql); } @Test public void testAsterisk4() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/asterisk_4.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/asterisk_4.sql"); parseQuery(sql); } @Test public void testGroupby1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/groupby_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/groupby_1.sql"); parseQuery(sql); } @Test public void testJoin1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_1.sql"); parseQuery(sql); } @Test public void testJoin2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_2.sql"); parseQuery(sql); } @Test public void testJoin3() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_3.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_3.sql"); parseQuery(sql); } @Test public void testJoin4() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_4.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_4.sql"); parseQuery(sql); } @Test public void testJoin5() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_5.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_5.sql"); parseQuery(sql); } @Test public void testJoin6() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_6.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_6.sql"); parseQuery(sql); } @Test public void testJoin7() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_7.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_7.sql"); parseQuery(sql); } @Test public void testJoin8() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_8.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_8.sql"); parseQuery(sql); } @Test public void testJoin9() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_9.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_9.sql"); parseQuery(sql); } @Test public void testJoin10() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_10.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_10.sql"); parseQuery(sql); } @Test public void testJoin11() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/join_11.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/join_11.sql"); parseQuery(sql); } @Test public void testSet1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/set_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/set_1.sql"); parseQuery(sql); } @Test public void testSet2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/set_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/set_2.sql"); parseQuery(sql); } @Test public void testSet3() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/set_3.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/set_3.sql"); parseQuery(sql); } @Test public void testSet4() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/set_4.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/set_4.sql"); parseQuery(sql); } @Test public void testDropTable() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/drop_table.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/drop_table.sql"); parseQuery(sql); } @Test public void testCreateTable1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_1.sql"); parseQuery(sql); } @Test public void testCreateTable2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_2.sql"); parseQuery(sql); } @Test public void testCreateTable3() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_3.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_3.sql"); parseQuery(sql); } @Test public void testCreateTable4() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_4.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_4.sql"); parseQuery(sql); } @Test public void testCreateTable5() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_5.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_5.sql"); parseQuery(sql); } @Test public void testCreateTable6() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_6.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_6.sql"); parseQuery(sql); } @Test public void testCreateTable7() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_7.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_7.sql"); parseQuery(sql); } @Test public void testCreateTable8() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_8.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_8.sql"); parseQuery(sql); } @Test public void testCreateTable9() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_9.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_9.sql"); parseQuery(sql); } @Test public void testCreateTable10() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_10.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_10.sql"); parseQuery(sql); } @Test public void testCreateTableLike1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_like_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_like_1.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.CreateTable, expr.getType()); CreateTable createTable = (CreateTable) expr; @@ -285,7 +285,7 @@ public void testCreateTableLike1() throws IOException { @Test public void testCreateTablePartitionByHash1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_partition_by_hash_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_partition_by_hash_1.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.CreateTable, expr.getType()); CreateTable createTable = (CreateTable) expr; @@ -298,7 +298,7 @@ public void testCreateTablePartitionByHash1() throws IOException { @Test public void testCreateTablePartitionByHash2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_partition_by_hash_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_partition_by_hash_2.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.CreateTable, expr.getType()); CreateTable createTable = (CreateTable) expr; @@ -312,7 +312,7 @@ public void testCreateTablePartitionByHash2() throws IOException { @Test public void testCreateTablePartitionByRange() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_partition_by_range.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_partition_by_range.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.CreateTable, expr.getType()); CreateTable createTable = (CreateTable) expr; @@ -325,7 +325,7 @@ public void testCreateTablePartitionByRange() throws IOException { @Test public void testCreateTablePartitionByList() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_partition_by_list.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_partition_by_list.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.CreateTable, expr.getType()); CreateTable createTable = (CreateTable) expr; @@ -350,7 +350,7 @@ public void testCreateTablePartitionByList() throws IOException { @Test public void testCreateTablePartitionByColumn() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/create_table_partition_by_column.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/create_table_partition_by_column.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.CreateTable, expr.getType()); CreateTable createTable = (CreateTable) expr; @@ -365,7 +365,7 @@ public void testCreateTablePartitionByColumn() throws IOException { @Test public void testAlterTableAddPartition1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/alter_table_add_partition_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_add_partition_1.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.AlterTable, expr.getType()); AlterTable alterTable = (AlterTable)expr; @@ -382,7 +382,7 @@ public void testAlterTableAddPartition1() throws IOException { @Test public void testAlterTableAddPartition2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/alter_table_add_partition_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_add_partition_2.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.AlterTable, expr.getType()); AlterTable alterTable = (AlterTable)expr; @@ -398,10 +398,44 @@ public void testAlterTableAddPartition2() throws IOException { assertEquals(alterTable.getLocation(), "hdfs://xxx.com/warehouse/table1/col1=1/col2=2"); } + @Test + public void testAlterTableAddPartition3() throws IOException { + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_add_partition_3.sql"); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); + assertEquals(3, alterTable.getColumns().length); + assertEquals(3, alterTable.getValues().length); + assertEquals("col1", alterTable.getColumns()[0].getName()); + assertEquals("col2", alterTable.getColumns()[1].getName()); + assertEquals("col3", alterTable.getColumns()[2].getName()); + LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; + assertEquals("2015", value1.getValue()); + LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; + assertEquals("01", value2.getValue()); + LiteralValue value3 = (LiteralValue)alterTable.getValues()[2]; + assertEquals("11", value3.getValue()); + assertEquals(alterTable.getLocation(), "hdfs://xxx.com/warehouse/table1/col1=2015/col2=01/col3=11"); + } + + @Test + public void testAlterTableAddPartition4() throws IOException { + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_add_partition_4.sql"); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.ADD_PARTITION); + assertEquals(1, alterTable.getColumns().length); + assertEquals(1, alterTable.getValues().length); + assertEquals("col1", alterTable.getColumns()[0].getName()); + LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; + assertEquals("TAJO", value1.getValue()); + } @Test public void testAlterTableDropPartition1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/alter_table_drop_partition_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_drop_partition_1.sql"); Expr expr = parseQuery(sql); assertEquals(OpType.AlterTable, expr.getType()); AlterTable alterTable = (AlterTable)expr; @@ -416,75 +450,109 @@ public void testAlterTableDropPartition1() throws IOException { assertEquals("2", value2.getValue()); } + @Test + public void testAlterTableDropPartition2() throws IOException { + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_drop_partition_2.sql"); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.DROP_PARTITION); + assertEquals(3, alterTable.getColumns().length); + assertEquals(3, alterTable.getValues().length); + assertEquals("col1", alterTable.getColumns()[0].getName()); + assertEquals("col2", alterTable.getColumns()[1].getName()); + assertEquals("col3", alterTable.getColumns()[2].getName()); + LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; + assertEquals("2015", value1.getValue()); + LiteralValue value2 = (LiteralValue)alterTable.getValues()[1]; + assertEquals("01", value2.getValue()); + LiteralValue value3 = (LiteralValue)alterTable.getValues()[2]; + assertEquals("11", value3.getValue()); + } + + @Test + public void testAlterTableDropPartition3() throws IOException { + String sql = FileUtil.readTextFileFromResource("queries/default/alter_table_drop_partition_3.sql"); + Expr expr = parseQuery(sql); + assertEquals(OpType.AlterTable, expr.getType()); + AlterTable alterTable = (AlterTable)expr; + assertEquals(alterTable.getAlterTableOpType(), AlterTableOpType.DROP_PARTITION); + assertEquals(1, alterTable.getColumns().length); + assertEquals(1, alterTable.getValues().length); + assertEquals("col1", alterTable.getColumns()[0].getName()); + LiteralValue value1 = (LiteralValue)alterTable.getValues()[0]; + assertEquals("TAJO", value1.getValue()); + } + @Test public void testTableSubQuery1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/table_subquery1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/table_subquery1.sql"); parseQuery(sql); } @Test public void testTableSubQuery2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/table_subquery2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/table_subquery2.sql"); parseQuery(sql); } @Test public void testInSubquery1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/in_subquery_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/in_subquery_1.sql"); parseQuery(sql); } @Test public void testInSubquery2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/in_subquery_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/in_subquery_2.sql"); parseQuery(sql); } @Test public void testExistsPredicate1() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/exists_predicate_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/exists_predicate_1.sql"); parseQuery(sql); } @Test public void testExistsPredicate2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/exists_predicate_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/exists_predicate_2.sql"); parseQuery(sql); } @Test public void testInsertIntoTable() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/insert_into_select_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/insert_into_select_1.sql"); parseQuery(sql); } @Test public void testInsertIntoLocation() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/insert_into_select_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/insert_into_select_2.sql"); parseQuery(sql); } @Test public void testInsertIntoTable2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/insert_into_select_3.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/insert_into_select_3.sql"); parseQuery(sql); } @Test public void testInsertOverwriteIntoTable() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/insert_overwrite_into_select_1.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/insert_overwrite_into_select_1.sql"); parseQuery(sql); } @Test public void testInsertOverwriteIntoLocation() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/insert_overwrite_into_select_2.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/insert_overwrite_into_select_2.sql"); parseQuery(sql); } @Test public void testInsertOverwriteIntoTable2() throws IOException { - String sql = FileUtil.readTextFile(new File("src/test/resources/queries/default/insert_overwrite_into_select_3.sql")); + String sql = FileUtil.readTextFileFromResource("queries/default/insert_overwrite_into_select_3.sql"); parseQuery(sql); } diff --git a/tajo-core/src/test/resources/queries/default/alter_table_add_partition_3.sql b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_3.sql new file mode 100644 index 0000000000..79d14266ce --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_3.sql @@ -0,0 +1,2 @@ +ALTER TABLE table1 ADD PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) +LOCATION 'hdfs://xxx.com/warehouse/table1/col1=2015/col2=01/col3=11' \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/default/alter_table_add_partition_4.sql b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_4.sql new file mode 100644 index 0000000000..bc22a1418f --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_add_partition_4.sql @@ -0,0 +1 @@ +ALTER TABLE table1 ADD PARTITION (col1 = 'TAJO' ) \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_2.sql b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_2.sql new file mode 100644 index 0000000000..57527107c6 --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_2.sql @@ -0,0 +1 @@ +ALTER TABLE table1 DROP PARTITION (col1 = '2015' , col2 = '01', col3 = '11' ) \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql new file mode 100644 index 0000000000..1942e16889 --- /dev/null +++ b/tajo-core/src/test/resources/queries/default/alter_table_drop_partition_3.sql @@ -0,0 +1 @@ +ALTER TABLE table1 DROP PARTITION (col1 = 'TAJO' ) \ No newline at end of file From dafd9983ce2fd01883bff36d3a918fcb0a62baf7 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 23 Jan 2015 18:06:07 +0900 Subject: [PATCH 3/4] Added PreLogicalPlanVerifier::visitAlterTable --- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 23 ++++++++++ .../org/apache/tajo/jdbc/TestTajoJdbc.java | 42 +++++++++++++++++++ .../testAlterTableAddPartition.result | 2 + .../testAlterTableDropPartition.result | 2 + .../plan/verifier/PreLogicalPlanVerifier.java | 12 ++++++ 5 files changed, 81 insertions(+) create mode 100644 tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result create mode 100644 tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index aff16774c7..4b0e021c28 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -27,6 +27,7 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.TpchTestBase; +import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; import org.apache.tajo.conf.TajoConf; @@ -385,6 +386,28 @@ public void run() { } } + @Test + public void testAlterTableAddPartition() throws Exception { + String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition"); + + tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); + tajoCli.executeScript("alter table " + tableName + " add partition (key = 0.1)"); + + String consoleResult = new String(out.toByteArray()); + assertOutputResult(consoleResult); + } + + @Test + public void testAlterTableDropPartition() throws Exception { + String tableName = CatalogUtil.normalizeIdentifier("testAlterTableDropPartition"); + + tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); + tajoCli.executeScript("alter table " + tableName + " drop partition (key = 0.1)"); + + String consoleResult = new String(out.toByteArray()); + assertOutputResult(consoleResult); + } + public static class TajoCliOutputTestFormatter extends DefaultTajoCliOutputFormatter { @Override protected String getResponseTimeReadable(float responseTime) { diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index 1c763e2d8f..0025299a84 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -593,4 +593,46 @@ public void testSortWithDateTime() throws Exception { } } } + + + @Test + public void testAlterTableAddPartition() throws Exception { + Statement stmt = null; + ResultSet resultSet = null; + int retCode = 0; + Connection conn = null; + int result; + String errorMessage = null; + + // skip this test if catalog uses HCatalogStore. + // It is because HCatalogStore does not support Time data type. + try { + if (!testingCluster.isHCatalogStoreRunning()) { + String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), + DEFAULT_DATABASE_NAME); + conn = DriverManager.getConnection(connUri); + assertTrue(conn.isValid(100)); + + String tableName = CatalogUtil.normalizeIdentifier("testAlterTablePartition"); + resultSet = executeString( + "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) "); + resultSet.close(); + + stmt = conn.createStatement(); + resultSet = stmt.executeQuery("alter table " + tableName + " add partition (key = 0.1)"); + } + } catch (SQLFeatureNotSupportedException e) { + errorMessage = e.getMessage(); + } finally { + assertEquals(errorMessage, "ADD_PARTITION is not supported yet\n"); + cleanupQuery(resultSet); + if (stmt != null) { + stmt.close(); + } + + if(conn != null) { + conn.close(); + } + } + } } \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result new file mode 100644 index 0000000000..31f46bc4f3 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableAddPartition.result @@ -0,0 +1,2 @@ +OK +ERROR: ADD_PARTITION is not supported yet \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result new file mode 100644 index 0000000000..1fadcea500 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoCli/testAlterTableDropPartition.result @@ -0,0 +1,2 @@ +OK +ERROR: DROP_PARTITION is not supported yet \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index c184fff94a..25452de4d1 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -334,4 +334,16 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws return expr; } + + @Override + public Expr visitAlterTable(Context context, Stack stack, AlterTable expr) throws PlanningException { + super.visitAlterTable(context, stack, expr); + + if (expr.getAlterTableOpType() == AlterTableOpType.ADD_PARTITION + || expr.getAlterTableOpType() == AlterTableOpType.DROP_PARTITION) { + context.state.addVerification(expr.getAlterTableOpType().name() + " is not supported yet"); + } + + return expr; + } } From ec543bfc007ce491ed6c48bc84b8795c62e55203 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 4 Feb 2015 16:22:34 +0900 Subject: [PATCH 4/4] Change error type. --- tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index 0025299a84..db6192cb97 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -621,7 +621,7 @@ public void testAlterTableAddPartition() throws Exception { stmt = conn.createStatement(); resultSet = stmt.executeQuery("alter table " + tableName + " add partition (key = 0.1)"); } - } catch (SQLFeatureNotSupportedException e) { + } catch (SQLException e) { errorMessage = e.getMessage(); } finally { assertEquals(errorMessage, "ADD_PARTITION is not supported yet\n");