Skip to content

HIVE-30015: Multiple identical Projects on top of HiveJdbcConverter #6757

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
soumyakanti3578 wants to merge 1 commit into
base: master
Choose a base branch
from
+261 −1
Draft

HIVE-30015: Multiple identical Projects on top of HiveJdbcConverter #6757

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...c/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ private static void convertOpTree(RelNode rel, RelNode parent) {
introduceDerivedTable(rel.getInput(0), rel);
}
} else if (rel instanceof SingleRel) {
if (rel instanceof HiveJdbcConverter) {
if (rel instanceof HiveJdbcConverter && !(parent instanceof Project)) {
introduceDerivedTable(rel, parent);
} else if (rel instanceof Filter) {
if (!validFilterParent(rel, parent)) {
Expand Down
50 changes: 50 additions & 0 deletions ql/src/test/queries/clientpositive/jdbc_redundant_project.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
--! qt:database:postgres:qdb:q_test_country_table_with_schema.postgres.sql

CREATE EXTERNAL TABLE country_bob (id int, name varchar(20))
STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "${system:hive.test.database.qdb.jdbc.url}",
"hive.sql.dbcp.username" = "${system:hive.test.database.qdb.jdbc.username}",
"hive.sql.dbcp.password" = "${system:hive.test.database.qdb.jdbc.password}",
"hive.sql.schema" = "bob",
"hive.sql.table" = "country");

CREATE EXTERNAL TABLE country_alice (id int, name varchar(20))
STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "${system:hive.test.database.qdb.jdbc.url}",
"hive.sql.dbcp.username" = "${system:hive.test.database.qdb.jdbc.username}",
"hive.sql.dbcp.password" = "${system:hive.test.database.qdb.jdbc.password}",
"hive.sql.schema" = "alice",
"hive.sql.table" = "country");

EXPLAIN CBO
WITH cross_items AS (
SELECT id
FROM (
SELECT id FROM country_bob
INTERSECT
SELECT id FROM country_alice
INTERSECT
SELECT id FROM country_bob
) x
)
SELECT channel, cnt
FROM (
SELECT 'a' channel, COUNT(*) cnt
FROM country_bob
WHERE id IN (SELECT id FROM cross_items)
UNION ALL
SELECT 'b' channel, COUNT(*) cnt
FROM country_alice
WHERE id IN (SELECT id FROM cross_items)
UNION ALL
SELECT 'c' channel, COUNT(*) cnt
FROM country_bob
WHERE id IN (SELECT id FROM cross_items)
) y
ORDER BY channel;
210 changes: 210 additions & 0 deletions ql/src/test/results/clientpositive/llap/jdbc_redundant_project.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
PREHOOK: query: CREATE EXTERNAL TABLE country_bob (id int, name varchar(20))
STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qdb",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "bob",
"hive.sql.table" = "country")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@country_bob
POSTHOOK: query: CREATE EXTERNAL TABLE country_bob (id int, name varchar(20))
STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qdb",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "bob",
"hive.sql.table" = "country")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@country_bob
PREHOOK: query: CREATE EXTERNAL TABLE country_alice (id int, name varchar(20))
STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qdb",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "alice",
"hive.sql.table" = "country")
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@country_alice
POSTHOOK: query: CREATE EXTERNAL TABLE country_alice (id int, name varchar(20))
STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qdb",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "alice",
"hive.sql.table" = "country")
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@country_alice
PREHOOK: query: EXPLAIN CBO
WITH cross_items AS (
SELECT id
FROM (
SELECT id FROM country_bob
INTERSECT
SELECT id FROM country_alice
INTERSECT
SELECT id FROM country_bob
) x
)
SELECT channel, cnt
FROM (
SELECT 'a' channel, COUNT(*) cnt
FROM country_bob
WHERE id IN (SELECT id FROM cross_items)
UNION ALL
SELECT 'b' channel, COUNT(*) cnt
FROM country_alice
WHERE id IN (SELECT id FROM cross_items)
UNION ALL
SELECT 'c' channel, COUNT(*) cnt
FROM country_bob
WHERE id IN (SELECT id FROM cross_items)
) y
ORDER BY channel
PREHOOK: type: QUERY
PREHOOK: Input: default@country_alice
PREHOOK: Input: default@country_bob
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO
WITH cross_items AS (
SELECT id
FROM (
SELECT id FROM country_bob
INTERSECT
SELECT id FROM country_alice
INTERSECT
SELECT id FROM country_bob
) x
)
SELECT channel, cnt
FROM (
SELECT 'a' channel, COUNT(*) cnt
FROM country_bob
WHERE id IN (SELECT id FROM cross_items)
UNION ALL
SELECT 'b' channel, COUNT(*) cnt
FROM country_alice
WHERE id IN (SELECT id FROM cross_items)
UNION ALL
SELECT 'c' channel, COUNT(*) cnt
FROM country_bob
WHERE id IN (SELECT id FROM cross_items)
) y
ORDER BY channel
POSTHOOK: type: QUERY
POSTHOOK: Input: default@country_alice
POSTHOOK: Input: default@country_bob
#### A masked pattern was here ####
CBO PLAN:
HiveSortLimit(sort0=[$0], dir0=[ASC])
HiveProject(channel=[$0], cnt=[$1])
HiveUnion(all=[true])
HiveProject(channel=['a':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"], cnt=[$0])
HiveAggregate(group=[{}], agg#0=[count()])
HiveSemiJoin(condition=[=($0, $1)], joinType=[semi])
HiveProject(id=[$0])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob])
HiveProject($f0=[$0])
HiveFilter(condition=[=($1, 3)])
HiveAggregate(group=[{0}], agg#0=[count($1)])
HiveProject(id=[$0], $f1=[$1])
HiveUnion(all=[true])
HiveProject(id=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{0}], agg#0=[count()])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob])
HiveProject(id=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{0}], agg#0=[count()])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_alice]], table:alias=[country_alice])
HiveProject(id=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{0}], agg#0=[count()])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob])
HiveProject(channel=['b':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"], cnt=[$0])
HiveAggregate(group=[{}], agg#0=[count()])
HiveSemiJoin(condition=[=($0, $1)], joinType=[semi])
HiveProject(id=[$0])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_alice]], table:alias=[country_alice])
HiveProject($f0=[$0])
HiveFilter(condition=[=($1, 3)])
HiveAggregate(group=[{0}], agg#0=[count($1)])
HiveProject(id=[$0], $f1=[$1])
HiveUnion(all=[true])
HiveProject(id=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{0}], agg#0=[count()])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob])
HiveProject(id=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{0}], agg#0=[count()])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_alice]], table:alias=[country_alice])
HiveProject(id=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{0}], agg#0=[count()])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob])
HiveProject(channel=['c':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"], cnt=[$0])
HiveAggregate(group=[{}], agg#0=[count()])
HiveSemiJoin(condition=[=($0, $1)], joinType=[semi])
HiveProject(id=[$0])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob])
HiveProject($f0=[$0])
HiveFilter(condition=[=($1, 3)])
HiveAggregate(group=[{0}], agg#0=[count($1)])
HiveProject(id=[$0], $f1=[$1])
HiveUnion(all=[true])
HiveProject(id=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{0}], agg#0=[count()])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob])
HiveProject(id=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{0}], agg#0=[count()])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_alice]], table:alias=[country_alice])
HiveProject(id=[$0], $f1=[$1])
HiveJdbcConverter(convention=[JDBC.POSTGRES])
JdbcAggregate(group=[{0}], agg#0=[count()])
JdbcFilter(condition=[IS NOT NULL($0)])
JdbcProject(id=[$0])
JdbcHiveTableScan(table=[[default, country_bob]], table:alias=[country_bob])

Loading