Skip to content

[SPARK-56814][SQL][TESTS] Add lateral join tests for outer attribute visibility after NATURAL/USING JOIN #55794

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mihailotim-db wants to merge 1 commit into from
+241 −0
Closed

[SPARK-56814][SQL][TESTS] Add lateral join tests for outer attribute visibility after NATURAL/USING JOIN #55794

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 130 additions & 0 deletions sql/core/src/test/resources/sql-tests/analyzer-results/join-lateral.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3064,6 +3064,136 @@ Project [1 AS 1#x]
+- LocalRelation [col1#x, col2#x]


-- !query
WITH nj1(k, v1) AS (VALUES (1, 'a')),
nj2(k, v2) AS (VALUES (1, 'b'))
SELECT * FROM nj1 NATURAL JOIN nj2,
LATERAL (SELECT k AS unq_k)
-- !query analysis
WithCTE
:- CTERelationDef xxxx, false
: +- SubqueryAlias nj1
: +- Project [col1#x AS k#x, col2#x AS v1#x]
: +- LocalRelation [col1#x, col2#x]
:- CTERelationDef xxxx, false
: +- SubqueryAlias nj2
: +- Project [col1#x AS k#x, col2#x AS v2#x]
: +- LocalRelation [col1#x, col2#x]
+- Project [k#x, v1#x, v2#x, unq_k#x]
+- LateralJoin lateral-subquery#x [k#x], Inner
: +- SubqueryAlias __auto_generated_subquery_name
: +- Project [outer(k#x) AS unq_k#x]
: +- OneRowRelation
+- Project [k#x, v1#x, v2#x]
+- Join Inner, (k#x = k#x)
:- SubqueryAlias nj1
: +- CTERelationRef xxxx, true, [k#x, v1#x], false, false, 1
+- SubqueryAlias nj2
+- CTERelationRef xxxx, true, [k#x, v2#x], false, false, 1


-- !query
WITH nj1(k, v1) AS (VALUES (1, 'a')),
nj2(k, v2) AS (VALUES (1, 'b'))
SELECT * FROM nj1 NATURAL JOIN nj2,
LATERAL (SELECT k AS unq_k, nj1.k AS qual_nj1k, nj2.k AS qual_nj2k)
-- !query analysis
WithCTE
:- CTERelationDef xxxx, false
: +- SubqueryAlias nj1
: +- Project [col1#x AS k#x, col2#x AS v1#x]
: +- LocalRelation [col1#x, col2#x]
:- CTERelationDef xxxx, false
: +- SubqueryAlias nj2
: +- Project [col1#x AS k#x, col2#x AS v2#x]
: +- LocalRelation [col1#x, col2#x]
+- Project [k#x, v1#x, v2#x, unq_k#x, qual_nj1k#x, qual_nj2k#x]
+- Project [k#x, v1#x, v2#x, unq_k#x, qual_nj1k#x, qual_nj2k#x]
+- LateralJoin lateral-subquery#x [k#x && k#x && k#x], Inner
: +- SubqueryAlias __auto_generated_subquery_name
: +- Project [outer(k#x) AS unq_k#x, outer(k#x) AS qual_nj1k#x, outer(k#x) AS qual_nj2k#x]
: +- OneRowRelation
+- Project [k#x, v1#x, v2#x, k#x]
+- Join Inner, (k#x = k#x)
:- SubqueryAlias nj1
: +- CTERelationRef xxxx, true, [k#x, v1#x], false, false, 1
+- SubqueryAlias nj2
+- CTERelationRef xxxx, true, [k#x, v2#x], false, false, 1


-- !query
WITH uj1(k, v1) AS (VALUES (1, 'a')),
uj2(k, v2) AS (VALUES (1, 'b'))
SELECT * FROM uj1 JOIN uj2 USING (k),
LATERAL (SELECT k AS unq_k, uj1.k AS qual_uj1k, uj2.k AS qual_uj2k)
-- !query analysis
WithCTE
:- CTERelationDef xxxx, false
: +- SubqueryAlias uj1
: +- Project [col1#x AS k#x, col2#x AS v1#x]
: +- LocalRelation [col1#x, col2#x]
:- CTERelationDef xxxx, false
: +- SubqueryAlias uj2
: +- Project [col1#x AS k#x, col2#x AS v2#x]
: +- LocalRelation [col1#x, col2#x]
+- Project [k#x, v1#x, v2#x, unq_k#x, qual_uj1k#x, qual_uj2k#x]
+- Project [k#x, v1#x, v2#x, unq_k#x, qual_uj1k#x, qual_uj2k#x]
+- LateralJoin lateral-subquery#x [k#x && k#x && k#x], Inner
: +- SubqueryAlias __auto_generated_subquery_name
: +- Project [outer(k#x) AS unq_k#x, outer(k#x) AS qual_uj1k#x, outer(k#x) AS qual_uj2k#x]
: +- OneRowRelation
+- Project [k#x, v1#x, v2#x, k#x]
+- Join Inner, (k#x = k#x)
:- SubqueryAlias uj1
: +- CTERelationRef xxxx, true, [k#x, v1#x], false, false, 1
+- SubqueryAlias uj2
+- CTERelationRef xxxx, true, [k#x, v2#x], false, false, 1


-- !query
WITH cte1(k, v1) AS (VALUES (1, 'a'))
SELECT * FROM (SELECT k FROM cte1 ORDER BY v1) sub,
LATERAL (SELECT v1 AS leaked)
-- !query analysis
org.apache.spark.sql.catalyst.ExtendedAnalysisException
{
"errorClass" : "UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
"sqlState" : "42703",
"messageParameters" : {
"objectName" : "`v1`"
},
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 107,
"stopIndex" : 108,
"fragment" : "v1"
} ]
}


-- !query
WITH cte1(k, v1) AS (VALUES (1, 'a'), (2, 'b'), (3, 'c'))
SELECT * FROM (SELECT k FROM cte1 GROUP BY k) g,
LATERAL (SELECT v1 AS leaked)
-- !query analysis
org.apache.spark.sql.catalyst.ExtendedAnalysisException
{
"errorClass" : "UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
"sqlState" : "42703",
"messageParameters" : {
"objectName" : "`v1`"
},
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 124,
"stopIndex" : 125,
"fragment" : "v1"
} ]
}


-- !query
DROP VIEW t1
-- !query analysis
Expand Down
30 changes: 30 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/join-lateral.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,36 @@ left join
order by t_inner.b1,t_inner.b2 desc limit 1
) as lateral_table;

-- lateral join after NATURAL/USING JOIN: outer attribute visibility

-- lateral after NATURAL JOIN: unqualified key resolves to the merged column
WITH nj1(k, v1) AS (VALUES (1, 'a')),
nj2(k, v2) AS (VALUES (1, 'b'))
SELECT * FROM nj1 NATURAL JOIN nj2,
LATERAL (SELECT k AS unq_k);

-- lateral after NATURAL JOIN: qualified keys resolve to original columns
WITH nj1(k, v1) AS (VALUES (1, 'a')),
nj2(k, v2) AS (VALUES (1, 'b'))
SELECT * FROM nj1 NATURAL JOIN nj2,
LATERAL (SELECT k AS unq_k, nj1.k AS qual_nj1k, nj2.k AS qual_nj2k);

-- lateral after USING JOIN: unqualified and qualified keys
WITH uj1(k, v1) AS (VALUES (1, 'a')),
uj2(k, v2) AS (VALUES (1, 'b'))
SELECT * FROM uj1 JOIN uj2 USING (k),
LATERAL (SELECT k AS unq_k, uj1.k AS qual_uj1k, uj2.k AS qual_uj2k);

-- lateral cannot see column hidden by a subquery alias
WITH cte1(k, v1) AS (VALUES (1, 'a'))
SELECT * FROM (SELECT k FROM cte1 ORDER BY v1) sub,
LATERAL (SELECT v1 AS leaked);

-- lateral cannot see column not in GROUP BY output
WITH cte1(k, v1) AS (VALUES (1, 'a'), (2, 'b'), (3, 'c'))
SELECT * FROM (SELECT k FROM cte1 GROUP BY k) g,
LATERAL (SELECT v1 AS leaked);

-- clean up
DROP VIEW t1;
DROP VIEW t2;
Expand Down
81 changes: 81 additions & 0 deletions sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1905,6 +1905,87 @@ struct<1:int>
1


-- !query
WITH nj1(k, v1) AS (VALUES (1, 'a')),
nj2(k, v2) AS (VALUES (1, 'b'))
SELECT * FROM nj1 NATURAL JOIN nj2,
LATERAL (SELECT k AS unq_k)
-- !query schema
struct<k:int,v1:string,v2:string,unq_k:int>
-- !query output
1 a b 1


-- !query
WITH nj1(k, v1) AS (VALUES (1, 'a')),
nj2(k, v2) AS (VALUES (1, 'b'))
SELECT * FROM nj1 NATURAL JOIN nj2,
LATERAL (SELECT k AS unq_k, nj1.k AS qual_nj1k, nj2.k AS qual_nj2k)
-- !query schema
struct<k:int,v1:string,v2:string,unq_k:int,qual_nj1k:int,qual_nj2k:int>
-- !query output
1 a b 1 1 1


-- !query
WITH uj1(k, v1) AS (VALUES (1, 'a')),
uj2(k, v2) AS (VALUES (1, 'b'))
SELECT * FROM uj1 JOIN uj2 USING (k),
LATERAL (SELECT k AS unq_k, uj1.k AS qual_uj1k, uj2.k AS qual_uj2k)
-- !query schema
struct<k:int,v1:string,v2:string,unq_k:int,qual_uj1k:int,qual_uj2k:int>
-- !query output
1 a b 1 1 1


-- !query
WITH cte1(k, v1) AS (VALUES (1, 'a'))
SELECT * FROM (SELECT k FROM cte1 ORDER BY v1) sub,
LATERAL (SELECT v1 AS leaked)
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.ExtendedAnalysisException
{
"errorClass" : "UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
"sqlState" : "42703",
"messageParameters" : {
"objectName" : "`v1`"
},
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 107,
"stopIndex" : 108,
"fragment" : "v1"
} ]
}


-- !query
WITH cte1(k, v1) AS (VALUES (1, 'a'), (2, 'b'), (3, 'c'))
SELECT * FROM (SELECT k FROM cte1 GROUP BY k) g,
LATERAL (SELECT v1 AS leaked)
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.ExtendedAnalysisException
{
"errorClass" : "UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
"sqlState" : "42703",
"messageParameters" : {
"objectName" : "`v1`"
},
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
"startIndex" : 124,
"stopIndex" : 125,
"fragment" : "v1"
} ]
}


-- !query
DROP VIEW t1
-- !query schema
Expand Down