From d911f981fca0661067554939491ae74889d7b8b0 Mon Sep 17 00:00:00 2001 From: Soumyakanti Das Date: Thu, 3 Sep 2026 16:32:26 -0700 Subject: [PATCH] HIVE-30015: Multiple identical Projects on top of HiveJdbcConverter --- .../translator/PlanModifierForASTConv.java | 2 +- .../clientpositive/jdbc_redundant_project.q | 50 +++++ .../llap/jdbc_redundant_project.q.out | 210 ++++++++++++++++++ 3 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 ql/src/test/queries/clientpositive/jdbc_redundant_project.q create mode 100644 ql/src/test/results/clientpositive/llap/jdbc_redundant_project.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java index 52e9b3ad2a35..3b7569411ace 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java @@ -163,7 +163,7 @@ private static void convertOpTree(RelNode rel, RelNode parent) { introduceDerivedTable(rel.getInput(0), rel); } } else if (rel instanceof SingleRel) { - if (rel instanceof HiveJdbcConverter) { + if (rel instanceof HiveJdbcConverter && !(parent instanceof Project)) { introduceDerivedTable(rel, parent); } else if (rel instanceof Filter) { if (!validFilterParent(rel, parent)) { diff --git a/ql/src/test/queries/clientpositive/jdbc_redundant_project.q b/ql/src/test/queries/clientpositive/jdbc_redundant_project.q new file mode 100644 index 000000000000..aa725d8e988a --- /dev/null +++ b/ql/src/test/queries/clientpositive/jdbc_redundant_project.q @@ -0,0 +1,50 @@ +--! qt:database:postgres:qdb:q_test_country_table_with_schema.postgres.sql + +CREATE EXTERNAL TABLE country_bob (id int, name varchar(20)) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "POSTGRES", + "hive.sql.jdbc.driver" = "org.postgresql.Driver", + "hive.sql.jdbc.url" = "${system:hive.test.database.qdb.jdbc.url}", + "hive.sql.dbcp.username" = "${system:hive.test.database.qdb.jdbc.username}", + "hive.sql.dbcp.password" = "${system:hive.test.database.qdb.jdbc.password}", + "hive.sql.schema" = "bob", + "hive.sql.table" = "country"); + +CREATE EXTERNAL TABLE country_alice (id int, name varchar(20)) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "POSTGRES", + "hive.sql.jdbc.driver" = "org.postgresql.Driver", + "hive.sql.jdbc.url" = "${system:hive.test.database.qdb.jdbc.url}", + "hive.sql.dbcp.username" = "${system:hive.test.database.qdb.jdbc.username}", + "hive.sql.dbcp.password" = "${system:hive.test.database.qdb.jdbc.password}", + "hive.sql.schema" = "alice", + "hive.sql.table" = "country"); + +EXPLAIN CBO +WITH cross_items AS ( + SELECT id + FROM ( + SELECT id FROM country_bob + INTERSECT + SELECT id FROM country_alice + INTERSECT + SELECT id FROM country_bob + ) x +) +SELECT channel, cnt +FROM ( + SELECT 'a' channel, COUNT(*) cnt + FROM country_bob + WHERE id IN (SELECT id FROM cross_items) + UNION ALL + SELECT 'b' channel, COUNT(*) cnt + FROM country_alice + WHERE id IN (SELECT id FROM cross_items) + UNION ALL + SELECT 'c' channel, COUNT(*) cnt + FROM country_bob + WHERE id IN (SELECT id FROM cross_items) +) y +ORDER BY channel; diff --git a/ql/src/test/results/clientpositive/llap/jdbc_redundant_project.q.out b/ql/src/test/results/clientpositive/llap/jdbc_redundant_project.q.out new file mode 100644 index 000000000000..fd6a6de7aa2b --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/jdbc_redundant_project.q.out @@ -0,0 +1,210 @@ +PREHOOK: query: CREATE EXTERNAL TABLE country_bob (id int, name varchar(20)) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "POSTGRES", + "hive.sql.jdbc.driver" = "org.postgresql.Driver", + "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qdb", + "hive.sql.dbcp.username" = "qtestuser", + "hive.sql.dbcp.password" = "qtestpassword", + "hive.sql.schema" = "bob", + "hive.sql.table" = "country") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@country_bob +POSTHOOK: query: CREATE EXTERNAL TABLE country_bob (id int, name varchar(20)) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "POSTGRES", + "hive.sql.jdbc.driver" = "org.postgresql.Driver", + "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qdb", + "hive.sql.dbcp.username" = "qtestuser", + "hive.sql.dbcp.password" = "qtestpassword", + "hive.sql.schema" = "bob", + "hive.sql.table" = "country") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@country_bob +PREHOOK: query: CREATE EXTERNAL TABLE country_alice (id int, name varchar(20)) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "POSTGRES", + "hive.sql.jdbc.driver" = "org.postgresql.Driver", + "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qdb", + "hive.sql.dbcp.username" = "qtestuser", + "hive.sql.dbcp.password" = "qtestpassword", + "hive.sql.schema" = "alice", + "hive.sql.table" = "country") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@country_alice +POSTHOOK: query: CREATE EXTERNAL TABLE country_alice (id int, name varchar(20)) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "POSTGRES", + "hive.sql.jdbc.driver" = "org.postgresql.Driver", + "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qdb", + "hive.sql.dbcp.username" = "qtestuser", + "hive.sql.dbcp.password" = "qtestpassword", + "hive.sql.schema" = "alice", + "hive.sql.table" = "country") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@country_alice +PREHOOK: query: EXPLAIN CBO +WITH cross_items AS ( + SELECT id + FROM ( + SELECT id FROM country_bob + INTERSECT + SELECT id FROM country_alice + INTERSECT + SELECT id FROM country_bob + ) x +) +SELECT channel, cnt +FROM ( + SELECT 'a' channel, COUNT(*) cnt + FROM country_bob + WHERE id IN (SELECT id FROM cross_items) + UNION ALL + SELECT 'b' channel, COUNT(*) cnt + FROM country_alice + WHERE id IN (SELECT id FROM cross_items) + UNION ALL + SELECT 'c' channel, COUNT(*) cnt + FROM country_bob + WHERE id IN (SELECT id FROM cross_items) +) y +ORDER BY channel +PREHOOK: type: QUERY +PREHOOK: Input: default@country_alice +PREHOOK: Input: default@country_bob +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN CBO +WITH cross_items AS ( + SELECT id + FROM ( + SELECT id FROM country_bob + INTERSECT + SELECT id FROM country_alice + INTERSECT + SELECT id FROM country_bob + ) x +) +SELECT channel, cnt +FROM ( + SELECT 'a' channel, COUNT(*) cnt + FROM country_bob + WHERE id IN (SELECT id FROM cross_items) + UNION ALL + SELECT 'b' channel, COUNT(*) cnt + FROM country_alice + WHERE id IN (SELECT id FROM cross_items) + UNION ALL + SELECT 'c' channel, COUNT(*) cnt + FROM country_bob + WHERE id IN (SELECT id FROM cross_items) +) y +ORDER BY channel +POSTHOOK: type: QUERY +POSTHOOK: Input: default@country_alice +POSTHOOK: Input: default@country_bob +#### A masked pattern was here #### +CBO PLAN: +HiveSortLimit(sort0=[$0], dir0=[ASC]) + HiveProject(channel=[$0], cnt=[$1]) + HiveUnion(all=[true]) + HiveProject(channel=['a':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"], cnt=[$0]) + HiveAggregate(group=[{}], agg#0=[count()]) + HiveSemiJoin(condition=[=($0, $1)], joinType=[semi]) + HiveProject(id=[$0]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob]) + HiveProject($f0=[$0]) + HiveFilter(condition=[=($1, 3)]) + HiveAggregate(group=[{0}], agg#0=[count($1)]) + HiveProject(id=[$0], $f1=[$1]) + HiveUnion(all=[true]) + HiveProject(id=[$0], $f1=[$1]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcAggregate(group=[{0}], agg#0=[count()]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob]) + HiveProject(id=[$0], $f1=[$1]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcAggregate(group=[{0}], agg#0=[count()]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_alice]], table:alias=[country_alice]) + HiveProject(id=[$0], $f1=[$1]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcAggregate(group=[{0}], agg#0=[count()]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob]) + HiveProject(channel=['b':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"], cnt=[$0]) + HiveAggregate(group=[{}], agg#0=[count()]) + HiveSemiJoin(condition=[=($0, $1)], joinType=[semi]) + HiveProject(id=[$0]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_alice]], table:alias=[country_alice]) + HiveProject($f0=[$0]) + HiveFilter(condition=[=($1, 3)]) + HiveAggregate(group=[{0}], agg#0=[count($1)]) + HiveProject(id=[$0], $f1=[$1]) + HiveUnion(all=[true]) + HiveProject(id=[$0], $f1=[$1]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcAggregate(group=[{0}], agg#0=[count()]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob]) + HiveProject(id=[$0], $f1=[$1]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcAggregate(group=[{0}], agg#0=[count()]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_alice]], table:alias=[country_alice]) + HiveProject(id=[$0], $f1=[$1]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcAggregate(group=[{0}], agg#0=[count()]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob]) + HiveProject(channel=['c':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"], cnt=[$0]) + HiveAggregate(group=[{}], agg#0=[count()]) + HiveSemiJoin(condition=[=($0, $1)], joinType=[semi]) + HiveProject(id=[$0]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob]) + HiveProject($f0=[$0]) + HiveFilter(condition=[=($1, 3)]) + HiveAggregate(group=[{0}], agg#0=[count($1)]) + HiveProject(id=[$0], $f1=[$1]) + HiveUnion(all=[true]) + HiveProject(id=[$0], $f1=[$1]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcAggregate(group=[{0}], agg#0=[count()]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob]) + HiveProject(id=[$0], $f1=[$1]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcAggregate(group=[{0}], agg#0=[count()]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_alice]], table:alias=[country_alice]) + HiveProject(id=[$0], $f1=[$1]) + HiveJdbcConverter(convention=[JDBC.POSTGRES]) + JdbcAggregate(group=[{0}], agg#0=[count()]) + JdbcFilter(condition=[IS NOT NULL($0)]) + JdbcProject(id=[$0]) + JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob]) +