From 55151f580cddd597b7d2d8244fc457095c2f514b Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Wed, 3 Dec 2014 17:34:28 +0900 Subject: [PATCH] TAJO-1223: Wrong query verification against asterisk and more expressions in select list. --- .../tajo/engine/query/TestInsertQuery.java | 22 +++++++ .../lineitem_year_month_ddl.sql | 18 ++++++ .../load_to_lineitem_year_month.sql | 1 + ...testInsertOverwriteWithAsteriskAndMore.sql | 1 + ...tInsertOverwriteWithAsteriskAndMore.result | 7 ++ .../plan/verifier/PreLogicalPlanVerifier.java | 64 +++++++++++-------- 6 files changed, 87 insertions(+), 26 deletions(-) create mode 100644 tajo-core/src/test/resources/queries/TestInsertQuery/lineitem_year_month_ddl.sql create mode 100644 tajo-core/src/test/resources/queries/TestInsertQuery/load_to_lineitem_year_month.sql create mode 100644 tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithAsteriskAndMore.sql create mode 100644 tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskAndMore.result diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java index 117f18635c..cc7dced4bc 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java @@ -391,6 +391,28 @@ public final void testInsertOverwriteWithAsterisk() throws Exception { executeString("DROP TABLE full_table_csv PURGE"); } + @Test + public final void testInsertOverwriteWithAsteriskAndMore() throws Exception { + ResultSet res = executeFile("lineitem_year_month_ddl.sql"); + res.close(); + + CatalogService catalog = testingCluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(getCurrentDatabase(), "lineitem_year_month")); + + res = executeFile("load_to_lineitem_year_month.sql"); + res.close(); + TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "lineitem_year_month"); + if (!testingCluster.isHCatalogStoreRunning()) { + assertEquals(5, desc.getStats().getNumRows().intValue()); + } + + res = executeQuery(); + assertResultSet(res); + res.close(); + + executeString("DROP TABLE lineitem_year_month PURGE"); + } + @Test public final void testInsertOverwriteIntoSelect() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("insertoverwriteintoselect"); diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/lineitem_year_month_ddl.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/lineitem_year_month_ddl.sql new file mode 100644 index 0000000000..fb18ad8bea --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/lineitem_year_month_ddl.sql @@ -0,0 +1,18 @@ +create table lineitem_year_month ( + l_orderkey bigint, + l_partkey bigint, + l_suppkey bigint, + l_linenumber int, + l_quantity float8, + l_extendedprice float8, + l_discount float8, + l_tax float8, + l_returnflag text, + l_linestatus text, + l_shipdate text, + l_commitdate text, + l_receiptdate text, + l_shipinstruct text, + l_shipmode text, + l_comment text +) PARTITION BY COLUMN (year TEXT, MONTH TEXT); \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/load_to_lineitem_year_month.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/load_to_lineitem_year_month.sql new file mode 100644 index 0000000000..601af12ee5 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/load_to_lineitem_year_month.sql @@ -0,0 +1 @@ +INSERT INTO lineitem_year_month SELECT *, SUBSTR(l_shipdate, 1,4) as year, SUBSTR(l_shipdate, 6, 2) as month FROM default.lineitem; \ No newline at end of file diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithAsteriskAndMore.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithAsteriskAndMore.sql new file mode 100644 index 0000000000..07c904d720 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithAsteriskAndMore.sql @@ -0,0 +1 @@ +select * from lineitem_year_month; \ No newline at end of file diff --git a/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskAndMore.result b/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskAndMore.result new file mode 100644 index 0000000000..bb797ba31f --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskAndMore.result @@ -0,0 +1,7 @@ +l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment,year,MONTH +------------------------------- +3,3,6540,2,49.0,46796.47,0.1,0.0,R,F,1993-11-09,1993-12-20,1993-11-24,TAKE BACK RETURN,RAIL, unusual accounts. eve,1993,11 +3,2,1798,1,45.0,54058.05,0.06,0.0,R,F,1994-02-02,1994-01-04,1994-02-23,NONE,AIR,ongside of the furiously brave acco,1994,02 +1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the,1996,03 +1,1,7311,2,36.0,45983.16,0.09,0.06,N,O,1996-04-12,1996-02-28,1996-04-20,TAKE BACK RETURN,MAIL,ly final dependencies: slyly bold ,1996,04 +2,2,1191,1,38.0,44694.46,0.0,0.05,N,O,1997-01-28,1997-01-14,1997-02-02,TAKE BACK RETURN,RAIL,ven requests. deposits breach a,1997,01 \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 95e0f30aa9..f6d04ba2a0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -254,36 +254,48 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws if (child != null && child.getType() == OpType.Projection) { Projection projection = (Projection) child; - int projectColumnNum = projection.getNamedExprs().length; - if (expr.hasTargetColumns()) { - int targetColumnNum = expr.getTargetColumns().length; + // checking if at least one asterisk exists in target list + boolean includeAsterisk = false; + for (NamedExpr namedExpr : projection.getNamedExprs()) { + includeAsterisk |= namedExpr.getExpr().getType() == OpType.Asterisk; + } - if (targetColumnNum > projectColumnNum) { - context.state.addVerification("INSERT has more target columns than expressions"); - } else if (targetColumnNum < projectColumnNum) { - context.state.addVerification("INSERT has more expressions than target columns"); - } - } else { - if (expr.hasTableName()) { - String qualifiedName = expr.getTableName(); - if (TajoConstants.EMPTY_STRING.equals(CatalogUtil.extractQualifier(expr.getTableName()))) { - qualifiedName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), - expr.getTableName()); - } + // If one asterisk expression exists, we verify the match between the target exprs and output exprs. + // This verification will be in LogicalPlanVerifier. + if (!includeAsterisk) { + + int projectColumnNum = projection.getNamedExprs().length; + + if (expr.hasTargetColumns()) { + int targetColumnNum = expr.getTargetColumns().length; - TableDesc table = catalog.getTableDesc(qualifiedName); - if (table == null) { - context.state.addVerification(String.format("relation \"%s\" does not exist", qualifiedName)); - return null; + if (targetColumnNum > projectColumnNum) { + context.state.addVerification("INSERT has more target columns than expressions"); + } else if (targetColumnNum < projectColumnNum) { + context.state.addVerification("INSERT has more expressions than target columns"); } - if (table.hasPartition()) { - int columnSize = table.getSchema().getColumns().size(); - columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size(); - if (projectColumnNum < columnSize) { - context.state.addVerification("INSERT has smaller expressions than target columns"); - } else if (projectColumnNum > columnSize) { - context.state.addVerification("INSERT has more expressions than target columns"); + } else { + if (expr.hasTableName()) { + String qualifiedName = expr.getTableName(); + if (TajoConstants.EMPTY_STRING.equals(CatalogUtil.extractQualifier(expr.getTableName()))) { + qualifiedName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), + expr.getTableName()); + } + + TableDesc table = catalog.getTableDesc(qualifiedName); + if (table == null) { + context.state.addVerification(String.format("relation \"%s\" does not exist", qualifiedName)); + return null; + } + if (table.hasPartition()) { + int columnSize = table.getSchema().getColumns().size(); + columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size(); + if (projectColumnNum < columnSize) { + context.state.addVerification("INSERT has smaller expressions than target columns"); + } else if (projectColumnNum > columnSize) { + context.state.addVerification("INSERT has more expressions than target columns"); + } } } }