From ad2ad933054c51ffcee1205b40f267023b8bb08c Mon Sep 17 00:00:00 2001 From: Daniel Tenedorio Date: Thu, 26 Sep 2024 09:58:27 -0700 Subject: [PATCH 1/3] commit commit --- .../sql/catalyst/parser/SqlBaseParser.g4 | 1 + .../sql/catalyst/parser/AstBuilder.scala | 4 +- .../analyzer-results/pipe-operators.sql.out | 715 ++++++++++++++++++ .../sql-tests/inputs/pipe-operators.sql | 129 ++++ .../sql-tests/results/pipe-operators.sql.out | 425 +++++++++++ .../sql/execution/SparkSqlParserSuite.scala | 19 +- 6 files changed, 1284 insertions(+), 9 deletions(-) diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index 094f7f5315b80..224a1fd590f5e 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -1493,6 +1493,7 @@ version operatorPipeRightSide : selectClause | whereClause + | joinRelation ; // When `SQL_standard_keyword_behavior=true`, there are 2 kinds of keywords in Spark SQL. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 674005caaf1b2..6f245fa2801c3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -5889,7 +5889,9 @@ class AstBuilder extends DataTypeAstBuilder SubqueryAlias(SubqueryAlias.generateSubqueryName(), left) } withWhereClause(c, withSubqueryAlias) - }.get) + }.getOrElse(Option(ctx.joinRelation()).map { c => + withJoinRelation(c, left) + }.get)) } /** diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out index c44ce153a2f41..323d8fe0056a5 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out @@ -62,6 +62,112 @@ InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_d +- LocalRelation [col1#x, col2#x] +-- !query +create temporary view join_test_t1 as select * from values (1) as grouping(a) +-- !query analysis +CreateViewCommand `join_test_t1`, select * from values (1) as grouping(a), false, false, LocalTempView, UNSUPPORTED, true + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +create temporary view join_test_t2 as select * from values (1) as grouping(a) +-- !query analysis +CreateViewCommand `join_test_t2`, select * from values (1) as grouping(a), false, false, LocalTempView, UNSUPPORTED, true + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +create temporary view join_test_t3 as select * from values (1) as grouping(a) +-- !query analysis +CreateViewCommand `join_test_t3`, select * from values (1) as grouping(a), false, false, LocalTempView, UNSUPPORTED, true + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +create temporary view join_test_empty_table as select a from join_test_t2 where false +-- !query analysis +CreateViewCommand `join_test_empty_table`, select a from join_test_t2 where false, false, false, LocalTempView, UNSUPPORTED, true + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +create temporary view lateral_test_t1(c1, c2) + as values (0, 1), (1, 2) +-- !query analysis +CreateViewCommand `lateral_test_t1`, [(c1,None), (c2,None)], values (0, 1), (1, 2), false, false, LocalTempView, UNSUPPORTED, true + +- LocalRelation [col1#x, col2#x] + + +-- !query +create temporary view lateral_test_t2(c1, c2) + as values (0, 2), (0, 3) +-- !query analysis +CreateViewCommand `lateral_test_t2`, [(c1,None), (c2,None)], values (0, 2), (0, 3), false, false, LocalTempView, UNSUPPORTED, true + +- LocalRelation [col1#x, col2#x] + + +-- !query +create temporary view lateral_test_t3(c1, c2) + as values (0, array(0, 1)), (1, array(2)), (2, array()), (null, array(4)) +-- !query analysis +CreateViewCommand `lateral_test_t3`, [(c1,None), (c2,None)], values (0, array(0, 1)), (1, array(2)), (2, array()), (null, array(4)), false, false, LocalTempView, UNSUPPORTED, true + +- LocalRelation [col1#x, col2#x] + + +-- !query +create temporary view lateral_test_t4(c1, c2) + as values (0, 1), (0, 2), (1, 1), (1, 3) +-- !query analysis +CreateViewCommand `lateral_test_t4`, [(c1,None), (c2,None)], values (0, 1), (0, 2), (1, 1), (1, 3), false, false, LocalTempView, UNSUPPORTED, true + +- LocalRelation [col1#x, col2#x] + + +-- !query +create temporary view natural_join_test_t1 as select * from values + ("one", 1), ("two", 2), ("three", 3) as natural_join_test_t1(k, v1) +-- !query analysis +CreateViewCommand `natural_join_test_t1`, select * from values + ("one", 1), ("two", 2), ("three", 3) as natural_join_test_t1(k, v1), false, false, LocalTempView, UNSUPPORTED, true + +- Project [k#x, v1#x] + +- SubqueryAlias natural_join_test_t1 + +- LocalRelation [k#x, v1#x] + + +-- !query +create temporary view natural_join_test_t2 as select * from values + ("one", 1), ("two", 22), ("one", 5) as natural_join_test_t2(k, v2) +-- !query analysis +CreateViewCommand `natural_join_test_t2`, select * from values + ("one", 1), ("two", 22), ("one", 5) as natural_join_test_t2(k, v2), false, false, LocalTempView, UNSUPPORTED, true + +- Project [k#x, v2#x] + +- SubqueryAlias natural_join_test_t2 + +- LocalRelation [k#x, v2#x] + + +-- !query +create temporary view natural_join_test_t3 as select * from values + ("one", 4), ("two", 5), ("one", 6) as natural_join_test_t3(k, v3) +-- !query analysis +CreateViewCommand `natural_join_test_t3`, select * from values + ("one", 4), ("two", 5), ("one", 6) as natural_join_test_t3(k, v3), false, false, LocalTempView, UNSUPPORTED, true + +- Project [k#x, v3#x] + +- SubqueryAlias natural_join_test_t3 + +- LocalRelation [k#x, v3#x] + + -- !query table t |> select 1 as x @@ -569,6 +675,615 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException } +-- !query +table join_test_t1 +|> inner join join_test_empty_table +-- !query analysis +Join Inner +:- SubqueryAlias join_test_t1 +: +- View (`join_test_t1`, [a#x]) +: +- Project [cast(a#x as int) AS a#x] +: +- Project [a#x] +: +- SubqueryAlias grouping +: +- LocalRelation [a#x] ++- SubqueryAlias join_test_empty_table + +- View (`join_test_empty_table`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table join_test_t1 +|> cross join join_test_empty_table +-- !query analysis +Join Cross +:- SubqueryAlias join_test_t1 +: +- View (`join_test_t1`, [a#x]) +: +- Project [cast(a#x as int) AS a#x] +: +- Project [a#x] +: +- SubqueryAlias grouping +: +- LocalRelation [a#x] ++- SubqueryAlias join_test_empty_table + +- View (`join_test_empty_table`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table join_test_t1 +|> left outer join join_test_empty_table +-- !query analysis +Join LeftOuter +:- SubqueryAlias join_test_t1 +: +- View (`join_test_t1`, [a#x]) +: +- Project [cast(a#x as int) AS a#x] +: +- Project [a#x] +: +- SubqueryAlias grouping +: +- LocalRelation [a#x] ++- SubqueryAlias join_test_empty_table + +- View (`join_test_empty_table`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table join_test_t1 +|> right outer join join_test_empty_table +-- !query analysis +Join RightOuter +:- SubqueryAlias join_test_t1 +: +- View (`join_test_t1`, [a#x]) +: +- Project [cast(a#x as int) AS a#x] +: +- Project [a#x] +: +- SubqueryAlias grouping +: +- LocalRelation [a#x] ++- SubqueryAlias join_test_empty_table + +- View (`join_test_empty_table`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table join_test_t1 +|> full outer join join_test_empty_table using (a) +-- !query analysis +Project [coalesce(a#x, a#x) AS a#x] ++- Join FullOuter, (a#x = a#x) + :- SubqueryAlias join_test_t1 + : +- View (`join_test_t1`, [a#x]) + : +- Project [cast(a#x as int) AS a#x] + : +- Project [a#x] + : +- SubqueryAlias grouping + : +- LocalRelation [a#x] + +- SubqueryAlias join_test_empty_table + +- View (`join_test_empty_table`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table join_test_t1 +|> full outer join join_test_empty_table on (join_test_t1.a = join_test_empty_table.a) +-- !query analysis +Join FullOuter, (a#x = a#x) +:- SubqueryAlias join_test_t1 +: +- View (`join_test_t1`, [a#x]) +: +- Project [cast(a#x as int) AS a#x] +: +- Project [a#x] +: +- SubqueryAlias grouping +: +- LocalRelation [a#x] ++- SubqueryAlias join_test_empty_table + +- View (`join_test_empty_table`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table join_test_t1 +|> left semi join join_test_empty_table +-- !query analysis +Join LeftSemi +:- SubqueryAlias join_test_t1 +: +- View (`join_test_t1`, [a#x]) +: +- Project [cast(a#x as int) AS a#x] +: +- Project [a#x] +: +- SubqueryAlias grouping +: +- LocalRelation [a#x] ++- SubqueryAlias join_test_empty_table + +- View (`join_test_empty_table`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table join_test_t1 +|> left anti join join_test_empty_table +-- !query analysis +Join LeftAnti +:- SubqueryAlias join_test_t1 +: +- View (`join_test_t1`, [a#x]) +: +- Project [cast(a#x as int) AS a#x] +: +- Project [a#x] +: +- SubqueryAlias grouping +: +- LocalRelation [a#x] ++- SubqueryAlias join_test_empty_table + +- View (`join_test_empty_table`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +select * from join_test_t1 where true +|> inner join join_test_empty_table +-- !query analysis +Join Inner +:- Project [a#x] +: +- Filter true +: +- SubqueryAlias join_test_t1 +: +- View (`join_test_t1`, [a#x]) +: +- Project [cast(a#x as int) AS a#x] +: +- Project [a#x] +: +- SubqueryAlias grouping +: +- LocalRelation [a#x] ++- SubqueryAlias join_test_empty_table + +- View (`join_test_empty_table`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- Filter false + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +select 1 as x, 2 as y +|> inner join (select 1 as x, 4 as y) using (x) +-- !query analysis +Project [x#x, y#x, y#x] ++- Join Inner, (x#x = x#x) + :- Project [1 AS x#x, 2 AS y#x] + : +- OneRowRelation + +- SubqueryAlias __auto_generated_subquery_name + +- Project [1 AS x#x, 4 AS y#x] + +- OneRowRelation + + +-- !query +table join_test_t1 +|> inner join (join_test_t2 jt2 inner join join_test_t3 jt3 using (a)) using (a) +|> select a, join_test_t1.a, jt2.a, jt3.a +-- !query analysis +Project [a#x, a#x, a#x, a#x] ++- Project [a#x, a#x, a#x] + +- Join Inner, (a#x = a#x) + :- SubqueryAlias join_test_t1 + : +- View (`join_test_t1`, [a#x]) + : +- Project [cast(a#x as int) AS a#x] + : +- Project [a#x] + : +- SubqueryAlias grouping + : +- LocalRelation [a#x] + +- Project [a#x, a#x] + +- Join Inner, (a#x = a#x) + :- SubqueryAlias jt2 + : +- SubqueryAlias join_test_t2 + : +- View (`join_test_t2`, [a#x]) + : +- Project [cast(a#x as int) AS a#x] + : +- Project [a#x] + : +- SubqueryAlias grouping + : +- LocalRelation [a#x] + +- SubqueryAlias jt3 + +- SubqueryAlias join_test_t3 + +- View (`join_test_t3`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table join_test_t1 +|> inner join join_test_t2 tablesample (100 percent) jt2 using (a) +-- !query analysis +Project [a#x] ++- Join Inner, (a#x = a#x) + :- SubqueryAlias join_test_t1 + : +- View (`join_test_t1`, [a#x]) + : +- Project [cast(a#x as int) AS a#x] + : +- Project [a#x] + : +- SubqueryAlias grouping + : +- LocalRelation [a#x] + +- Sample 0.0, 1.0, false, 750 + +- SubqueryAlias jt2 + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table join_test_t1 +|> inner join (select 1 as a) tablesample (100 percent) jt2 using (a) +-- !query analysis +Project [a#x] ++- Join Inner, (a#x = a#x) + :- SubqueryAlias join_test_t1 + : +- View (`join_test_t1`, [a#x]) + : +- Project [cast(a#x as int) AS a#x] + : +- Project [a#x] + : +- SubqueryAlias grouping + : +- LocalRelation [a#x] + +- SubqueryAlias jt2 + +- Sample 0.0, 1.0, false, 77 + +- Project [1 AS a#x] + +- OneRowRelation + + +-- !query +table join_test_t1 +|> join join_test_t1 using (a) +-- !query analysis +Project [a#x] ++- Join Inner, (a#x = a#x) + :- SubqueryAlias join_test_t1 + : +- View (`join_test_t1`, [a#x]) + : +- Project [cast(a#x as int) AS a#x] + : +- Project [a#x] + : +- SubqueryAlias grouping + : +- LocalRelation [a#x] + +- SubqueryAlias join_test_t1 + +- View (`join_test_t1`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] + + +-- !query +table lateral_test_t1 +|> join lateral (select c1) +-- !query analysis +LateralJoin lateral-subquery#x [c1#x], Inner +: +- SubqueryAlias __auto_generated_subquery_name +: +- Project [outer(c1#x) AS c1#x] +: +- OneRowRelation ++- SubqueryAlias lateral_test_t1 + +- View (`lateral_test_t1`, [c1#x, c2#x]) + +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +table lateral_test_t1 +|> join lateral (select c1 from lateral_test_t2) +-- !query analysis +LateralJoin lateral-subquery#x [], Inner +: +- SubqueryAlias __auto_generated_subquery_name +: +- Project [c1#x] +: +- SubqueryAlias lateral_test_t2 +: +- View (`lateral_test_t2`, [c1#x, c2#x]) +: +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] +: +- LocalRelation [col1#x, col2#x] ++- SubqueryAlias lateral_test_t1 + +- View (`lateral_test_t1`, [c1#x, c2#x]) + +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +table lateral_test_t1 +|> join lateral (select lateral_test_t1.c1 from lateral_test_t2) +-- !query analysis +LateralJoin lateral-subquery#x [c1#x], Inner +: +- SubqueryAlias __auto_generated_subquery_name +: +- Project [outer(c1#x) AS c1#x] +: +- SubqueryAlias lateral_test_t2 +: +- View (`lateral_test_t2`, [c1#x, c2#x]) +: +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] +: +- LocalRelation [col1#x, col2#x] ++- SubqueryAlias lateral_test_t1 + +- View (`lateral_test_t1`, [c1#x, c2#x]) + +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +table lateral_test_t1 +|> join lateral (select lateral_test_t1.c1 + t2.c1 from lateral_test_t2 t2) +-- !query analysis +LateralJoin lateral-subquery#x [c1#x], Inner +: +- SubqueryAlias __auto_generated_subquery_name +: +- Project [(outer(c1#x) + c1#x) AS (outer(lateral_test_t1.c1) + c1)#x] +: +- SubqueryAlias t2 +: +- SubqueryAlias lateral_test_t2 +: +- View (`lateral_test_t2`, [c1#x, c2#x]) +: +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] +: +- LocalRelation [col1#x, col2#x] ++- SubqueryAlias lateral_test_t1 + +- View (`lateral_test_t1`, [c1#x, c2#x]) + +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +table lateral_test_t1 +|> join lateral (select *) +-- !query analysis +LateralJoin lateral-subquery#x [], Inner +: +- SubqueryAlias __auto_generated_subquery_name +: +- Project +: +- OneRowRelation ++- SubqueryAlias lateral_test_t1 + +- View (`lateral_test_t1`, [c1#x, c2#x]) + +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +table lateral_test_t1 +|> join lateral (select * from lateral_test_t2) +-- !query analysis +LateralJoin lateral-subquery#x [], Inner +: +- SubqueryAlias __auto_generated_subquery_name +: +- Project [c1#x, c2#x] +: +- SubqueryAlias lateral_test_t2 +: +- View (`lateral_test_t2`, [c1#x, c2#x]) +: +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] +: +- LocalRelation [col1#x, col2#x] ++- SubqueryAlias lateral_test_t1 + +- View (`lateral_test_t1`, [c1#x, c2#x]) + +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +table lateral_test_t1 +|> join lateral (select lateral_test_t1.* from lateral_test_t2) +-- !query analysis +LateralJoin lateral-subquery#x [c1#x && c2#x], Inner +: +- SubqueryAlias __auto_generated_subquery_name +: +- Project [outer(c1#x) AS c1#x, outer(c2#x) AS c2#x] +: +- SubqueryAlias lateral_test_t2 +: +- View (`lateral_test_t2`, [c1#x, c2#x]) +: +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] +: +- LocalRelation [col1#x, col2#x] ++- SubqueryAlias lateral_test_t1 + +- View (`lateral_test_t1`, [c1#x, c2#x]) + +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +table lateral_test_t1 +|> join lateral (select lateral_test_t1.*, t2.* from lateral_test_t2 t2) +-- !query analysis +LateralJoin lateral-subquery#x [c1#x && c2#x], Inner +: +- SubqueryAlias __auto_generated_subquery_name +: +- Project [outer(c1#x) AS c1#x, outer(c2#x) AS c2#x, c1#x, c2#x] +: +- SubqueryAlias t2 +: +- SubqueryAlias lateral_test_t2 +: +- View (`lateral_test_t2`, [c1#x, c2#x]) +: +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] +: +- LocalRelation [col1#x, col2#x] ++- SubqueryAlias lateral_test_t1 + +- View (`lateral_test_t1`, [c1#x, c2#x]) + +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +table lateral_test_t1 +|> join lateral_test_t2 +|> join lateral (select lateral_test_t1.c2 + lateral_test_t2.c2) +-- !query analysis +LateralJoin lateral-subquery#x [c2#x && c2#x], Inner +: +- SubqueryAlias __auto_generated_subquery_name +: +- Project [(outer(c2#x) + outer(c2#x)) AS (outer(lateral_test_t1.c2) + outer(lateral_test_t2.c2))#x] +: +- OneRowRelation ++- Join Inner + :- SubqueryAlias lateral_test_t1 + : +- View (`lateral_test_t1`, [c1#x, c2#x]) + : +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + : +- LocalRelation [col1#x, col2#x] + +- SubqueryAlias lateral_test_t2 + +- View (`lateral_test_t2`, [c1#x, c2#x]) + +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] + +- LocalRelation [col1#x, col2#x] + + +-- !query +table natural_join_test_t1 +|> natural join natural_join_test_t2 +|> where k = "one" +-- !query analysis +Filter (k#x = one) ++- SubqueryAlias __auto_generated_subquery_name + +- Project [k#x, v1#x, v2#x] + +- Join Inner, (k#x = k#x) + :- SubqueryAlias natural_join_test_t1 + : +- View (`natural_join_test_t1`, [k#x, v1#x]) + : +- Project [cast(k#x as string) AS k#x, cast(v1#x as int) AS v1#x] + : +- Project [k#x, v1#x] + : +- SubqueryAlias natural_join_test_t1 + : +- LocalRelation [k#x, v1#x] + +- SubqueryAlias natural_join_test_t2 + +- View (`natural_join_test_t2`, [k#x, v2#x]) + +- Project [cast(k#x as string) AS k#x, cast(v2#x as int) AS v2#x] + +- Project [k#x, v2#x] + +- SubqueryAlias natural_join_test_t2 + +- LocalRelation [k#x, v2#x] + + +-- !query +table natural_join_test_t1 +|> natural join natural_join_test_t2 nt2 +|> select natural_join_test_t1.* +-- !query analysis +Project [k#x, v1#x] ++- Project [k#x, v1#x, v2#x] + +- Join Inner, (k#x = k#x) + :- SubqueryAlias natural_join_test_t1 + : +- View (`natural_join_test_t1`, [k#x, v1#x]) + : +- Project [cast(k#x as string) AS k#x, cast(v1#x as int) AS v1#x] + : +- Project [k#x, v1#x] + : +- SubqueryAlias natural_join_test_t1 + : +- LocalRelation [k#x, v1#x] + +- SubqueryAlias nt2 + +- SubqueryAlias natural_join_test_t2 + +- View (`natural_join_test_t2`, [k#x, v2#x]) + +- Project [cast(k#x as string) AS k#x, cast(v2#x as int) AS v2#x] + +- Project [k#x, v2#x] + +- SubqueryAlias natural_join_test_t2 + +- LocalRelation [k#x, v2#x] + + +-- !query +table natural_join_test_t1 +|> natural join natural_join_test_t2 nt2 +|> natural join natural_join_test_t3 nt3 +|> select natural_join_test_t1.*, nt2.*, nt3.* +-- !query analysis +Project [k#x, v1#x, k#x, v2#x, k#x, v3#x] ++- Project [k#x, v1#x, v2#x, v3#x, k#x, k#x] + +- Join Inner, (k#x = k#x) + :- Project [k#x, v1#x, v2#x, k#x] + : +- Join Inner, (k#x = k#x) + : :- SubqueryAlias natural_join_test_t1 + : : +- View (`natural_join_test_t1`, [k#x, v1#x]) + : : +- Project [cast(k#x as string) AS k#x, cast(v1#x as int) AS v1#x] + : : +- Project [k#x, v1#x] + : : +- SubqueryAlias natural_join_test_t1 + : : +- LocalRelation [k#x, v1#x] + : +- SubqueryAlias nt2 + : +- SubqueryAlias natural_join_test_t2 + : +- View (`natural_join_test_t2`, [k#x, v2#x]) + : +- Project [cast(k#x as string) AS k#x, cast(v2#x as int) AS v2#x] + : +- Project [k#x, v2#x] + : +- SubqueryAlias natural_join_test_t2 + : +- LocalRelation [k#x, v2#x] + +- SubqueryAlias nt3 + +- SubqueryAlias natural_join_test_t3 + +- View (`natural_join_test_t3`, [k#x, v3#x]) + +- Project [cast(k#x as string) AS k#x, cast(v3#x as int) AS v3#x] + +- Project [k#x, v3#x] + +- SubqueryAlias natural_join_test_t3 + +- LocalRelation [k#x, v3#x] + + +-- !query +table join_test_t1 +|> inner join join_test_empty_table + inner join join_test_empty_table +-- !query analysis +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'inner'", + "hint" : "" + } +} + + +-- !query +table join_test_t1 +|> select 1 + 2 as result +|> full outer join join_test_empty_table on (join_test_t1.a = join_test_empty_table.a) +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNRESOLVED_COLUMN.WITH_SUGGESTION", + "sqlState" : "42703", + "messageParameters" : { + "objectName" : "`join_test_t1`.`a`", + "proposal" : "`result`, `join_test_empty_table`.`a`" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 91, + "stopIndex" : 104, + "fragment" : "join_test_t1.a" + } ] +} + + +-- !query +table join_test_t1 jt +|> cross join (select * from jt) +-- !query analysis +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'jt'", + "hint" : "" + } +} + + -- !query drop table t -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql b/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql index 49a72137ee047..341c162ecb67f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql @@ -12,6 +12,29 @@ drop table if exists st; create table st(x int, col struct) using parquet; insert into st values (1, (2, 3)); +create temporary view join_test_t1 as select * from values (1) as grouping(a); +create temporary view join_test_t2 as select * from values (1) as grouping(a); +create temporary view join_test_t3 as select * from values (1) as grouping(a); +create temporary view join_test_empty_table as select a from join_test_t2 where false; + +create temporary view lateral_test_t1(c1, c2) + as values (0, 1), (1, 2); +create temporary view lateral_test_t2(c1, c2) + as values (0, 2), (0, 3); +create temporary view lateral_test_t3(c1, c2) + as values (0, array(0, 1)), (1, array(2)), (2, array()), (null, array(4)); +create temporary view lateral_test_t4(c1, c2) + as values (0, 1), (0, 2), (1, 1), (1, 3); + +create temporary view natural_join_test_t1 as select * from values + ("one", 1), ("two", 2), ("three", 3) as natural_join_test_t1(k, v1); + +create temporary view natural_join_test_t2 as select * from values + ("one", 1), ("two", 22), ("one", 5) as natural_join_test_t2(k, v2); + +create temporary view natural_join_test_t3 as select * from values + ("one", 4), ("two", 5), ("one", 6) as natural_join_test_t3(k, v3); + -- SELECT operators: positive tests. --------------------------------------- @@ -185,6 +208,112 @@ table t (select x, sum(length(y)) as sum_len from t group by x) |> where sum(length(y)) = 3; +-- JOIN operators: positive tests. +---------------------------------- + +table join_test_t1 +|> inner join join_test_empty_table; + +table join_test_t1 +|> cross join join_test_empty_table; + +table join_test_t1 +|> left outer join join_test_empty_table; + +table join_test_t1 +|> right outer join join_test_empty_table; + +table join_test_t1 +|> full outer join join_test_empty_table using (a); + +table join_test_t1 +|> full outer join join_test_empty_table on (join_test_t1.a = join_test_empty_table.a); + +table join_test_t1 +|> left semi join join_test_empty_table; + +table join_test_t1 +|> left anti join join_test_empty_table; + +select * from join_test_t1 where true +|> inner join join_test_empty_table; + +select 1 as x, 2 as y +|> inner join (select 1 as x, 4 as y) using (x); + +table join_test_t1 +|> inner join (join_test_t2 jt2 inner join join_test_t3 jt3 using (a)) using (a) +|> select a, join_test_t1.a, jt2.a, jt3.a; + +table join_test_t1 +|> inner join join_test_t2 tablesample (100 percent) jt2 using (a); + +table join_test_t1 +|> inner join (select 1 as a) tablesample (100 percent) jt2 using (a); + +table join_test_t1 +|> join join_test_t1 using (a); + +-- Lateral joins. +table lateral_test_t1 +|> join lateral (select c1); + +table lateral_test_t1 +|> join lateral (select c1 from lateral_test_t2); + +table lateral_test_t1 +|> join lateral (select lateral_test_t1.c1 from lateral_test_t2); + +table lateral_test_t1 +|> join lateral (select lateral_test_t1.c1 + t2.c1 from lateral_test_t2 t2); + +table lateral_test_t1 +|> join lateral (select *); + +table lateral_test_t1 +|> join lateral (select * from lateral_test_t2); + +table lateral_test_t1 +|> join lateral (select lateral_test_t1.* from lateral_test_t2); + +table lateral_test_t1 +|> join lateral (select lateral_test_t1.*, t2.* from lateral_test_t2 t2); + +table lateral_test_t1 +|> join lateral_test_t2 +|> join lateral (select lateral_test_t1.c2 + lateral_test_t2.c2); + +-- Natural joins. +table natural_join_test_t1 +|> natural join natural_join_test_t2 +|> where k = "one"; + +table natural_join_test_t1 +|> natural join natural_join_test_t2 nt2 +|> select natural_join_test_t1.*; + +table natural_join_test_t1 +|> natural join natural_join_test_t2 nt2 +|> natural join natural_join_test_t3 nt3 +|> select natural_join_test_t1.*, nt2.*, nt3.*; + +-- JOIN operators: negative tests. +---------------------------------- + +-- Multiple joins within the same pipe operator are not supported without parentheses. +table join_test_t1 +|> inner join join_test_empty_table + inner join join_test_empty_table; + +-- The join pipe operator can only refer to column names from the previous relation. +table join_test_t1 +|> select 1 + 2 as result +|> full outer join join_test_empty_table on (join_test_t1.a = join_test_empty_table.a); + +-- The table from the pipe input is not visible as a table name in the right side. +table join_test_t1 jt +|> cross join (select * from jt); + -- Cleanup. ----------- drop table t; diff --git a/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out b/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out index 38436b0941034..a9b2492edf895 100644 --- a/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out @@ -71,6 +71,101 @@ struct<> +-- !query +create temporary view join_test_t1 as select * from values (1) as grouping(a) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view join_test_t2 as select * from values (1) as grouping(a) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view join_test_t3 as select * from values (1) as grouping(a) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view join_test_empty_table as select a from join_test_t2 where false +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view lateral_test_t1(c1, c2) + as values (0, 1), (1, 2) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view lateral_test_t2(c1, c2) + as values (0, 2), (0, 3) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view lateral_test_t3(c1, c2) + as values (0, array(0, 1)), (1, array(2)), (2, array()), (null, array(4)) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view lateral_test_t4(c1, c2) + as values (0, 1), (0, 2), (1, 1), (1, 3) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view natural_join_test_t1 as select * from values + ("one", 1), ("two", 2), ("three", 3) as natural_join_test_t1(k, v1) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view natural_join_test_t2 as select * from values + ("one", 1), ("two", 22), ("one", 5) as natural_join_test_t2(k, v2) +-- !query schema +struct<> +-- !query output + + + +-- !query +create temporary view natural_join_test_t3 as select * from values + ("one", 4), ("two", 5), ("one", 6) as natural_join_test_t3(k, v3) +-- !query schema +struct<> +-- !query output + + + -- !query table t |> select 1 as x @@ -552,6 +647,336 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException } +-- !query +table join_test_t1 +|> inner join join_test_empty_table +-- !query schema +struct +-- !query output + + + +-- !query +table join_test_t1 +|> cross join join_test_empty_table +-- !query schema +struct +-- !query output + + + +-- !query +table join_test_t1 +|> left outer join join_test_empty_table +-- !query schema +struct +-- !query output +1 NULL + + +-- !query +table join_test_t1 +|> right outer join join_test_empty_table +-- !query schema +struct +-- !query output + + + +-- !query +table join_test_t1 +|> full outer join join_test_empty_table using (a) +-- !query schema +struct +-- !query output +1 + + +-- !query +table join_test_t1 +|> full outer join join_test_empty_table on (join_test_t1.a = join_test_empty_table.a) +-- !query schema +struct +-- !query output +1 NULL + + +-- !query +table join_test_t1 +|> left semi join join_test_empty_table +-- !query schema +struct +-- !query output + + + +-- !query +table join_test_t1 +|> left anti join join_test_empty_table +-- !query schema +struct +-- !query output +1 + + +-- !query +select * from join_test_t1 where true +|> inner join join_test_empty_table +-- !query schema +struct +-- !query output + + + +-- !query +select 1 as x, 2 as y +|> inner join (select 1 as x, 4 as y) using (x) +-- !query schema +struct +-- !query output +1 2 4 + + +-- !query +table join_test_t1 +|> inner join (join_test_t2 jt2 inner join join_test_t3 jt3 using (a)) using (a) +|> select a, join_test_t1.a, jt2.a, jt3.a +-- !query schema +struct +-- !query output +1 1 1 1 + + +-- !query +table join_test_t1 +|> inner join join_test_t2 tablesample (100 percent) jt2 using (a) +-- !query schema +struct +-- !query output +1 + + +-- !query +table join_test_t1 +|> inner join (select 1 as a) tablesample (100 percent) jt2 using (a) +-- !query schema +struct +-- !query output +1 + + +-- !query +table join_test_t1 +|> join join_test_t1 using (a) +-- !query schema +struct +-- !query output +1 + + +-- !query +table lateral_test_t1 +|> join lateral (select c1) +-- !query schema +struct +-- !query output +0 1 0 +1 2 1 + + +-- !query +table lateral_test_t1 +|> join lateral (select c1 from lateral_test_t2) +-- !query schema +struct +-- !query output +0 1 0 +0 1 0 +1 2 0 +1 2 0 + + +-- !query +table lateral_test_t1 +|> join lateral (select lateral_test_t1.c1 from lateral_test_t2) +-- !query schema +struct +-- !query output +0 1 0 +0 1 0 +1 2 1 +1 2 1 + + +-- !query +table lateral_test_t1 +|> join lateral (select lateral_test_t1.c1 + t2.c1 from lateral_test_t2 t2) +-- !query schema +struct +-- !query output +0 1 0 +0 1 0 +1 2 1 +1 2 1 + + +-- !query +table lateral_test_t1 +|> join lateral (select *) +-- !query schema +struct +-- !query output +0 1 +1 2 + + +-- !query +table lateral_test_t1 +|> join lateral (select * from lateral_test_t2) +-- !query schema +struct +-- !query output +0 1 0 2 +0 1 0 3 +1 2 0 2 +1 2 0 3 + + +-- !query +table lateral_test_t1 +|> join lateral (select lateral_test_t1.* from lateral_test_t2) +-- !query schema +struct +-- !query output +0 1 0 1 +0 1 0 1 +1 2 1 2 +1 2 1 2 + + +-- !query +table lateral_test_t1 +|> join lateral (select lateral_test_t1.*, t2.* from lateral_test_t2 t2) +-- !query schema +struct +-- !query output +0 1 0 1 0 2 +0 1 0 1 0 3 +1 2 1 2 0 2 +1 2 1 2 0 3 + + +-- !query +table lateral_test_t1 +|> join lateral_test_t2 +|> join lateral (select lateral_test_t1.c2 + lateral_test_t2.c2) +-- !query schema +struct +-- !query output +0 1 0 2 3 +0 1 0 3 4 +1 2 0 2 4 +1 2 0 3 5 + + +-- !query +table natural_join_test_t1 +|> natural join natural_join_test_t2 +|> where k = "one" +-- !query schema +struct +-- !query output +one 1 1 +one 1 5 + + +-- !query +table natural_join_test_t1 +|> natural join natural_join_test_t2 nt2 +|> select natural_join_test_t1.* +-- !query schema +struct +-- !query output +one 1 +one 1 +two 2 + + +-- !query +table natural_join_test_t1 +|> natural join natural_join_test_t2 nt2 +|> natural join natural_join_test_t3 nt3 +|> select natural_join_test_t1.*, nt2.*, nt3.* +-- !query schema +struct +-- !query output +one 1 one 1 one 4 +one 1 one 1 one 6 +one 1 one 5 one 4 +one 1 one 5 one 6 +two 2 two 22 two 5 + + +-- !query +table join_test_t1 +|> inner join join_test_empty_table + inner join join_test_empty_table +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'inner'", + "hint" : "" + } +} + + +-- !query +table join_test_t1 +|> select 1 + 2 as result +|> full outer join join_test_empty_table on (join_test_t1.a = join_test_empty_table.a) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "UNRESOLVED_COLUMN.WITH_SUGGESTION", + "sqlState" : "42703", + "messageParameters" : { + "objectName" : "`join_test_t1`.`a`", + "proposal" : "`result`, `join_test_empty_table`.`a`" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 91, + "stopIndex" : 104, + "fragment" : "join_test_t1.a" + } ] +} + + +-- !query +table join_test_t1 jt +|> cross join (select * from jt) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'jt'", + "hint" : "" + } +} + + -- !query drop table t -- !query schema diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala index ab949c5a21e44..a8cc8c390b825 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedAlias, Un import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, Concat, GreaterThan, Literal, NullsFirst, SortOrder, UnresolvedWindowExpression, UnspecifiedFrame, WindowSpecDefinition, WindowSpecReference} import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.trees.TreePattern.{FILTER, LOCAL_RELATION, PROJECT, UNRESOLVED_RELATION} +import org.apache.spark.sql.catalyst.trees.TreePattern._ import org.apache.spark.sql.connector.catalog.TableCatalog import org.apache.spark.sql.execution.command._ import org.apache.spark.sql.execution.datasources.{CreateTempViewUsing, RefreshResource} @@ -887,24 +887,27 @@ class SparkSqlParserSuite extends AnalysisTest with SharedSparkSession { // Basic selection. // Here we check that every parsed plan contains a projection and a source relation or // inline table. - def checkPipeSelect(query: String): Unit = { + def check(query: String, patterns: Seq[TreePattern]): Unit = { val plan: LogicalPlan = parser.parsePlan(query) - assert(plan.containsPattern(PROJECT)) + assert(patterns.exists(plan.containsPattern)) assert(plan.containsAnyPattern(UNRESOLVED_RELATION, LOCAL_RELATION)) } + def checkPipeSelect(query: String): Unit = check(query, Seq(PROJECT)) checkPipeSelect("TABLE t |> SELECT 1 AS X") checkPipeSelect("TABLE t |> SELECT 1 AS X, 2 AS Y |> SELECT X + Y AS Z") checkPipeSelect("VALUES (0), (1) tab(col) |> SELECT col * 2 AS result") // Basic WHERE operators. - def checkPipeWhere(query: String): Unit = { - val plan: LogicalPlan = parser.parsePlan(query) - assert(plan.containsPattern(FILTER)) - assert(plan.containsAnyPattern(UNRESOLVED_RELATION, LOCAL_RELATION)) - } + def checkPipeWhere(query: String): Unit = check(query, Seq(FILTER)) checkPipeWhere("TABLE t |> WHERE X = 1") checkPipeWhere("TABLE t |> SELECT X, LENGTH(Y) AS Z |> WHERE X + LENGTH(Y) < 4") checkPipeWhere("TABLE t |> WHERE X = 1 AND Y = 2 |> WHERE X + Y = 3") checkPipeWhere("VALUES (0), (1) tab(col) |> WHERE col < 1") + // Joins. + def checkPipeJoin(query: String): Unit = check(query, Seq(JOIN)) + Seq("", "INNER", "LEFT", "LEFT OUTER", "SEMI", "LEFT SEMI", "RIGHT", "RIGHT OUTER", "FULL", + "FULL OUTER", "ANTI", "LEFT ANTI", "CROSS").foreach { joinType => + checkPipeJoin(s"TABLE t |> $joinType JOIN other ON (t.x = other.x)") + } } } } From 5d444fbc9aff1417498ac965d24c654b1f37410e Mon Sep 17 00:00:00 2001 From: Daniel Tenedorio Date: Fri, 4 Oct 2024 20:24:36 -0700 Subject: [PATCH 2/3] fix test --- .../analyzer-results/pipe-operators.sql.out | 50 ++++++++----------- .../sql-tests/inputs/pipe-operators.sql | 4 +- .../sql-tests/results/pipe-operators.sql.out | 28 ++++++++--- 3 files changed, 44 insertions(+), 38 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out index 317f300a9bf91..6dcb345b02494 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out @@ -1477,42 +1477,32 @@ Project [a#x, a#x, a#x, a#x] -- !query table join_test_t1 -|> inner join join_test_t2 tablesample (100 percent) jt2 using (a) +|> inner join join_test_t2 tablesample (100 percent repeatable 0) jt2 using (a) -- !query analysis -Project [a#x] -+- Join Inner, (a#x = a#x) - :- SubqueryAlias join_test_t1 - : +- View (`join_test_t1`, [a#x]) - : +- Project [cast(a#x as int) AS a#x] - : +- Project [a#x] - : +- SubqueryAlias grouping - : +- LocalRelation [a#x] - +- Sample 0.0, 1.0, false, 750 - +- SubqueryAlias jt2 - +- SubqueryAlias join_test_t2 - +- View (`join_test_t2`, [a#x]) - +- Project [cast(a#x as int) AS a#x] - +- Project [a#x] - +- SubqueryAlias grouping - +- LocalRelation [a#x] +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'repeatable'", + "hint" : ": missing ')'" + } +} -- !query table join_test_t1 -|> inner join (select 1 as a) tablesample (100 percent) jt2 using (a) +|> inner join (select 1 as a) tablesample (100 percent repeatable 0) jt2 using (a) -- !query analysis -Project [a#x] -+- Join Inner, (a#x = a#x) - :- SubqueryAlias join_test_t1 - : +- View (`join_test_t1`, [a#x]) - : +- Project [cast(a#x as int) AS a#x] - : +- Project [a#x] - : +- SubqueryAlias grouping - : +- LocalRelation [a#x] - +- SubqueryAlias jt2 - +- Sample 0.0, 1.0, false, 77 - +- Project [1 AS a#x] - +- OneRowRelation +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'repeatable'", + "hint" : ": missing ')'" + } +} -- !query diff --git a/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql b/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql index 2249699b6af1a..0f0b7aea4a13a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql @@ -436,10 +436,10 @@ table join_test_t1 |> select a, join_test_t1.a, jt2.a, jt3.a; table join_test_t1 -|> inner join join_test_t2 tablesample (100 percent) jt2 using (a); +|> inner join join_test_t2 tablesample (100 percent repeatable 0) jt2 using (a); table join_test_t1 -|> inner join (select 1 as a) tablesample (100 percent) jt2 using (a); +|> inner join (select 1 as a) tablesample (100 percent repeatable 0) jt2 using (a); table join_test_t1 |> join join_test_t1 using (a); diff --git a/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out b/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out index bd5effed9d86b..084962bbeb6c1 100644 --- a/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out @@ -1256,20 +1256,36 @@ struct -- !query table join_test_t1 -|> inner join join_test_t2 tablesample (100 percent) jt2 using (a) +|> inner join join_test_t2 tablesample (100 percent repeatable 0) jt2 using (a) -- !query schema -struct +struct<> -- !query output -1 +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'repeatable'", + "hint" : ": missing ')'" + } +} -- !query table join_test_t1 -|> inner join (select 1 as a) tablesample (100 percent) jt2 using (a) +|> inner join (select 1 as a) tablesample (100 percent repeatable 0) jt2 using (a) -- !query schema -struct +struct<> -- !query output -1 +org.apache.spark.sql.catalyst.parser.ParseException +{ + "errorClass" : "PARSE_SYNTAX_ERROR", + "sqlState" : "42601", + "messageParameters" : { + "error" : "'repeatable'", + "hint" : ": missing ')'" + } +} -- !query From e8ba2228f357b441a416a2bfc68cd153570a547e Mon Sep 17 00:00:00 2001 From: Daniel Tenedorio Date: Fri, 4 Oct 2024 20:33:15 -0700 Subject: [PATCH 3/3] fix test --- .../analyzer-results/pipe-operators.sql.out | 56 +++++++++++-------- .../sql-tests/inputs/pipe-operators.sql | 6 +- .../sql-tests/results/pipe-operators.sql.out | 34 +++-------- 3 files changed, 45 insertions(+), 51 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out index 6dcb345b02494..4479c93f6e84e 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/pipe-operators.sql.out @@ -1088,7 +1088,7 @@ org.apache.spark.sql.catalyst.parser.ParseException -- !query table t -|> tablesample (-100 percent) +|> tablesample (-100 percent) repeatable (0) -- !query analysis org.apache.spark.sql.catalyst.parser.ParseException { @@ -1100,8 +1100,8 @@ org.apache.spark.sql.catalyst.parser.ParseException "objectType" : "", "objectName" : "", "startIndex" : 12, - "stopIndex" : 37, - "fragment" : "tablesample (-100 percent)" + "stopIndex" : 52, + "fragment" : "tablesample (-100 percent) repeatable (0)" } ] } @@ -1477,32 +1477,42 @@ Project [a#x, a#x, a#x, a#x] -- !query table join_test_t1 -|> inner join join_test_t2 tablesample (100 percent repeatable 0) jt2 using (a) +|> inner join join_test_t2 tablesample (100 percent) repeatable (0) jt2 using (a) -- !query analysis -org.apache.spark.sql.catalyst.parser.ParseException -{ - "errorClass" : "PARSE_SYNTAX_ERROR", - "sqlState" : "42601", - "messageParameters" : { - "error" : "'repeatable'", - "hint" : ": missing ')'" - } -} +Project [a#x] ++- Join Inner, (a#x = a#x) + :- SubqueryAlias join_test_t1 + : +- View (`join_test_t1`, [a#x]) + : +- Project [cast(a#x as int) AS a#x] + : +- Project [a#x] + : +- SubqueryAlias grouping + : +- LocalRelation [a#x] + +- Sample 0.0, 1.0, false, 0 + +- SubqueryAlias jt2 + +- SubqueryAlias join_test_t2 + +- View (`join_test_t2`, [a#x]) + +- Project [cast(a#x as int) AS a#x] + +- Project [a#x] + +- SubqueryAlias grouping + +- LocalRelation [a#x] -- !query table join_test_t1 -|> inner join (select 1 as a) tablesample (100 percent repeatable 0) jt2 using (a) +|> inner join (select 1 as a) tablesample (100 percent) repeatable (0) jt2 using (a) -- !query analysis -org.apache.spark.sql.catalyst.parser.ParseException -{ - "errorClass" : "PARSE_SYNTAX_ERROR", - "sqlState" : "42601", - "messageParameters" : { - "error" : "'repeatable'", - "hint" : ": missing ')'" - } -} +Project [a#x] ++- Join Inner, (a#x = a#x) + :- SubqueryAlias join_test_t1 + : +- View (`join_test_t1`, [a#x]) + : +- Project [cast(a#x as int) AS a#x] + : +- Project [a#x] + : +- SubqueryAlias grouping + : +- LocalRelation [a#x] + +- SubqueryAlias jt2 + +- Sample 0.0, 1.0, false, 0 + +- Project [1 AS a#x] + +- OneRowRelation -- !query diff --git a/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql b/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql index 0f0b7aea4a13a..1f8450e3507cb 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pipe-operators.sql @@ -377,7 +377,7 @@ table t -- Negative sampling options are not supported. table t -|> tablesample (-100 percent); +|> tablesample (-100 percent) repeatable (0); table t |> tablesample (-5 rows); @@ -436,10 +436,10 @@ table join_test_t1 |> select a, join_test_t1.a, jt2.a, jt3.a; table join_test_t1 -|> inner join join_test_t2 tablesample (100 percent repeatable 0) jt2 using (a); +|> inner join join_test_t2 tablesample (100 percent) repeatable (0) jt2 using (a); table join_test_t1 -|> inner join (select 1 as a) tablesample (100 percent repeatable 0) jt2 using (a); +|> inner join (select 1 as a) tablesample (100 percent) repeatable (0) jt2 using (a); table join_test_t1 |> join join_test_t1 using (a); diff --git a/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out b/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out index 084962bbeb6c1..a54e66e53f0f3 100644 --- a/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pipe-operators.sql.out @@ -1019,7 +1019,7 @@ org.apache.spark.sql.catalyst.parser.ParseException -- !query table t -|> tablesample (-100 percent) +|> tablesample (-100 percent) repeatable (0) -- !query schema struct<> -- !query output @@ -1033,8 +1033,8 @@ org.apache.spark.sql.catalyst.parser.ParseException "objectType" : "", "objectName" : "", "startIndex" : 12, - "stopIndex" : 37, - "fragment" : "tablesample (-100 percent)" + "stopIndex" : 52, + "fragment" : "tablesample (-100 percent) repeatable (0)" } ] } @@ -1256,36 +1256,20 @@ struct -- !query table join_test_t1 -|> inner join join_test_t2 tablesample (100 percent repeatable 0) jt2 using (a) +|> inner join join_test_t2 tablesample (100 percent) repeatable (0) jt2 using (a) -- !query schema -struct<> +struct -- !query output -org.apache.spark.sql.catalyst.parser.ParseException -{ - "errorClass" : "PARSE_SYNTAX_ERROR", - "sqlState" : "42601", - "messageParameters" : { - "error" : "'repeatable'", - "hint" : ": missing ')'" - } -} +1 -- !query table join_test_t1 -|> inner join (select 1 as a) tablesample (100 percent repeatable 0) jt2 using (a) +|> inner join (select 1 as a) tablesample (100 percent) repeatable (0) jt2 using (a) -- !query schema -struct<> +struct -- !query output -org.apache.spark.sql.catalyst.parser.ParseException -{ - "errorClass" : "PARSE_SYNTAX_ERROR", - "sqlState" : "42601", - "messageParameters" : { - "error" : "'repeatable'", - "hint" : ": missing ')'" - } -} +1 -- !query