Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Spark3: test cases for Table-Valued Functions #2703

Merged
merged 2 commits into from Feb 27, 2022
Merged

Spark3: test cases for Table-Valued Functions #2703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
34c2261
test cases for table valued functions
R7L208 Feb 22, 2022
269c06e
Merge branch 'main' into r7l208/spark3-table-valued-function
tunetheweb Feb 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 19 additions & 0 deletions test/fixtures/dialects/spark3/from_supported_tvf.sql
View file
Open in desktop
@@ -0,0 +1,19 @@
--TVFs that are supported in a `FROM` clause
--
-- range call with end
SELECT id FROM range(6 + cos(3));
SELECT id FROM range(5);

-- range call with start and end
SELECT id FROM range(5, 10);

-- range call with start, end and step
SELECT id FROM range(5, 10, 2);

-- range call with start, end, step, and numPartitions
SELECT id FROM range(0, 10, 2, 200);

-- range call with a table alias
SELECT test.id FROM range(5, 8) AS test;

SELECT test.id FROM range(5, 8) test;
196 changes: 196 additions & 0 deletions test/fixtures/dialects/spark3/from_supported_tvf.yml
View file
Open in desktop
@@ -0,0 +1,196 @@
# YML test files are auto-generated from SQL files and should not be edited by
# hand. To help enforce this, the "hash" field in the file must match a hash
# computed by SQLFluff when running the tests. Please run
# `python test/generate_parse_fixture_yml.py` to generate them after adding or
# altering SQL files.
_hash: 673adfe5390702ee95279dfd1c4ee42f78d9142f43b05ca5a1d2be0ac7442d76
file:
- statement:
select_statement:
select_clause:
keyword: SELECT
select_clause_element:
column_reference:
identifier: id
from_clause:
keyword: FROM
from_expression:
from_expression_element:
table_expression:
function:
function_name:
function_name_identifier: range
bracketed:
start_bracket: (
expression:
literal: '6'
binary_operator: +
function:
function_name:
function_name_identifier: cos
bracketed:
start_bracket: (
expression:
literal: '3'
end_bracket: )
end_bracket: )
- statement_terminator: ;
- statement:
select_statement:
select_clause:
keyword: SELECT
select_clause_element:
column_reference:
identifier: id
from_clause:
keyword: FROM
from_expression:
from_expression_element:
table_expression:
function:
function_name:
function_name_identifier: range
bracketed:
start_bracket: (
expression:
literal: '5'
end_bracket: )
- statement_terminator: ;
- statement:
select_statement:
select_clause:
keyword: SELECT
select_clause_element:
column_reference:
identifier: id
from_clause:
keyword: FROM
from_expression:
from_expression_element:
table_expression:
function:
function_name:
function_name_identifier: range
bracketed:
- start_bracket: (
- expression:
literal: '5'
- comma: ','
- expression:
literal: '10'
- end_bracket: )
- statement_terminator: ;
- statement:
select_statement:
select_clause:
keyword: SELECT
select_clause_element:
column_reference:
identifier: id
from_clause:
keyword: FROM
from_expression:
from_expression_element:
table_expression:
function:
function_name:
function_name_identifier: range
bracketed:
- start_bracket: (
- expression:
literal: '5'
- comma: ','
- expression:
literal: '10'
- comma: ','
- expression:
literal: '2'
- end_bracket: )
- statement_terminator: ;
- statement:
select_statement:
select_clause:
keyword: SELECT
select_clause_element:
column_reference:
identifier: id
from_clause:
keyword: FROM
from_expression:
from_expression_element:
table_expression:
function:
function_name:
function_name_identifier: range
bracketed:
- start_bracket: (
- expression:
literal: '0'
- comma: ','
- expression:
literal: '10'
- comma: ','
- expression:
literal: '2'
- comma: ','
- expression:
literal: '200'
- end_bracket: )
- statement_terminator: ;
- statement:
select_statement:
select_clause:
keyword: SELECT
select_clause_element:
column_reference:
- identifier: test
- dot: .
- identifier: id
from_clause:
keyword: FROM
from_expression:
from_expression_element:
table_expression:
function:
function_name:
function_name_identifier: range
bracketed:
- start_bracket: (
- expression:
literal: '5'
- comma: ','
- expression:
literal: '8'
- end_bracket: )
alias_expression:
keyword: AS
identifier: test
- statement_terminator: ;
- statement:
select_statement:
select_clause:
keyword: SELECT
select_clause_element:
column_reference:
- identifier: test
- dot: .
- identifier: id
from_clause:
keyword: FROM
from_expression:
from_expression_element:
table_expression:
function:
function_name:
function_name_identifier: range
bracketed:
- start_bracket: (
- expression:
literal: '5'
- comma: ','
- expression:
literal: '8'
- end_bracket: )
alias_expression:
identifier: test
- statement_terminator: ;
93 changes: 93 additions & 0 deletions test/fixtures/dialects/spark3/select_lateral_view_supported_tvf.sql
View file
Open in desktop
@@ -0,0 +1,93 @@
-- TVFs that can be specified in SELECT/LATERAL VIEW clauses

-- explode in a SELECT
SELECT explode(array(10, 20));

-- explode_outer in a SELECT
SELECT explode_outer(array(10, 20));

-- explode in a LATERAL VIEW clause
SELECT
test.a,
test.b
FROM test
LATERAL VIEW explode(array(3, 4)) AS c2;

-- explode_outer in a LATERAL VIEW clause
SELECT
test.a,
test.b
FROM test
LATERAL VIEW explode_outer(array(3, 4)) AS c2;

-- inline in a SELECT
SELECT inline(array(struct(1, 'a'), struct(2, 'b')));

-- inline_outer in a SELECT
SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b')));

-- inline in a LATERAL VIEW clause
SELECT
test.a,
test.b
FROM test
LATERAL VIEW inline(array(struct(1, 'a'), struct(2, 'b'))) AS c1, c2;

-- inline_outer in a LATERAL VIEW clause
SELECT
test.a,
test.b
FROM test
LATERAL VIEW inline_outer(array(struct(1, 'a'), struct(2, 'b'))) AS c1, c2;

-- posexplode in a SELECT
SELECT posexplode(array(10, 20));

-- posexplode_outer in a SELECT
SELECT posexplode_outer(array(10, 20));

-- posexplode in a LATERAL VIEW clause
SELECT
test.a,
test.b
FROM test
LATERAL VIEW posexplode(array(10, 20)) AS c1;

-- posexplode_outer in a LATERAL VIEW clause
SELECT
test.a,
test.b
FROM test
LATERAL VIEW posexplode_outer(array(10, 20)) AS c1;

-- stack in a SELECT
SELECT stack(2, 1, 2, 3);

-- stack in a LATERAL VIEW clause
SELECT
test.a,
test.b
FROM test
LATERAL VIEW stack(2, 1, 2, 3) AS c1, c2;

-- json_tuple in a SELECT
SELECT json_tuple('{"a":1, "b":2}', 'a', 'b');

-- json_tuple in a LATERAL VIEW clause
SELECT
test.a,
test.b
FROM test
LATERAL VIEW json_tuple('{"a":1, "b":2}', 'a', 'b') AS c1, c2;

-- parse_url in a SELECT
SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST');

-- parse_url in a LATERAL VIEW clause
SELECT
test.a,
test.b
FROM test
LATERAL VIEW parse_url(
'http://spark.apache.org/path?query=1', 'HOST'
) AS c1;