From cce8d551ab161151e0c510fc796e4991f64e403d Mon Sep 17 00:00:00 2001 From: Lorin Dawson <22798188+R7L208@users.noreply.github.com> Date: Sun, 27 Feb 2022 05:21:46 -0500 Subject: [PATCH] test cases for table valued functions (#2703) Co-authored-by: Barry Pollard --- .../dialects/spark3/from_supported_tvf.sql | 19 + .../dialects/spark3/from_supported_tvf.yml | 196 +++++ .../select_lateral_view_supported_tvf.sql | 93 +++ .../select_lateral_view_supported_tvf.yml | 717 ++++++++++++++++++ 4 files changed, 1025 insertions(+) create mode 100644 test/fixtures/dialects/spark3/from_supported_tvf.sql create mode 100644 test/fixtures/dialects/spark3/from_supported_tvf.yml create mode 100644 test/fixtures/dialects/spark3/select_lateral_view_supported_tvf.sql create mode 100644 test/fixtures/dialects/spark3/select_lateral_view_supported_tvf.yml diff --git a/test/fixtures/dialects/spark3/from_supported_tvf.sql b/test/fixtures/dialects/spark3/from_supported_tvf.sql new file mode 100644 index 00000000000..bb6183c3a9c --- /dev/null +++ b/test/fixtures/dialects/spark3/from_supported_tvf.sql @@ -0,0 +1,19 @@ +--TVFs that are supported in a `FROM` clause +-- +-- range call with end +SELECT id FROM range(6 + cos(3)); +SELECT id FROM range(5); + +-- range call with start and end +SELECT id FROM range(5, 10); + +-- range call with start, end and step +SELECT id FROM range(5, 10, 2); + +-- range call with start, end, step, and numPartitions +SELECT id FROM range(0, 10, 2, 200); + +-- range call with a table alias +SELECT test.id FROM range(5, 8) AS test; + +SELECT test.id FROM range(5, 8) test; diff --git a/test/fixtures/dialects/spark3/from_supported_tvf.yml b/test/fixtures/dialects/spark3/from_supported_tvf.yml new file mode 100644 index 00000000000..f5966af6e36 --- /dev/null +++ b/test/fixtures/dialects/spark3/from_supported_tvf.yml @@ -0,0 +1,196 @@ +# YML test files are auto-generated from SQL files and should not be edited by +# hand. To help enforce this, the "hash" field in the file must match a hash +# computed by SQLFluff when running the tests. Please run +# `python test/generate_parse_fixture_yml.py` to generate them after adding or +# altering SQL files. +_hash: 673adfe5390702ee95279dfd1c4ee42f78d9142f43b05ca5a1d2be0ac7442d76 +file: +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + column_reference: + identifier: id + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + function: + function_name: + function_name_identifier: range + bracketed: + start_bracket: ( + expression: + literal: '6' + binary_operator: + + function: + function_name: + function_name_identifier: cos + bracketed: + start_bracket: ( + expression: + literal: '3' + end_bracket: ) + end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + column_reference: + identifier: id + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + function: + function_name: + function_name_identifier: range + bracketed: + start_bracket: ( + expression: + literal: '5' + end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + column_reference: + identifier: id + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + function: + function_name: + function_name_identifier: range + bracketed: + - start_bracket: ( + - expression: + literal: '5' + - comma: ',' + - expression: + literal: '10' + - end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + column_reference: + identifier: id + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + function: + function_name: + function_name_identifier: range + bracketed: + - start_bracket: ( + - expression: + literal: '5' + - comma: ',' + - expression: + literal: '10' + - comma: ',' + - expression: + literal: '2' + - end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + column_reference: + identifier: id + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + function: + function_name: + function_name_identifier: range + bracketed: + - start_bracket: ( + - expression: + literal: '0' + - comma: ',' + - expression: + literal: '10' + - comma: ',' + - expression: + literal: '2' + - comma: ',' + - expression: + literal: '200' + - end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: id + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + function: + function_name: + function_name_identifier: range + bracketed: + - start_bracket: ( + - expression: + literal: '5' + - comma: ',' + - expression: + literal: '8' + - end_bracket: ) + alias_expression: + keyword: AS + identifier: test +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: id + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + function: + function_name: + function_name_identifier: range + bracketed: + - start_bracket: ( + - expression: + literal: '5' + - comma: ',' + - expression: + literal: '8' + - end_bracket: ) + alias_expression: + identifier: test +- statement_terminator: ; diff --git a/test/fixtures/dialects/spark3/select_lateral_view_supported_tvf.sql b/test/fixtures/dialects/spark3/select_lateral_view_supported_tvf.sql new file mode 100644 index 00000000000..e9346e4b55e --- /dev/null +++ b/test/fixtures/dialects/spark3/select_lateral_view_supported_tvf.sql @@ -0,0 +1,93 @@ +-- TVFs that can be specified in SELECT/LATERAL VIEW clauses + +-- explode in a SELECT +SELECT explode(array(10, 20)); + +-- explode_outer in a SELECT +SELECT explode_outer(array(10, 20)); + +-- explode in a LATERAL VIEW clause +SELECT + test.a, + test.b +FROM test + LATERAL VIEW explode(array(3, 4)) AS c2; + +-- explode_outer in a LATERAL VIEW clause +SELECT + test.a, + test.b +FROM test + LATERAL VIEW explode_outer(array(3, 4)) AS c2; + +-- inline in a SELECT +SELECT inline(array(struct(1, 'a'), struct(2, 'b'))); + +-- inline_outer in a SELECT +SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))); + +-- inline in a LATERAL VIEW clause +SELECT + test.a, + test.b +FROM test + LATERAL VIEW inline(array(struct(1, 'a'), struct(2, 'b'))) AS c1, c2; + +-- inline_outer in a LATERAL VIEW clause +SELECT + test.a, + test.b +FROM test + LATERAL VIEW inline_outer(array(struct(1, 'a'), struct(2, 'b'))) AS c1, c2; + +-- posexplode in a SELECT +SELECT posexplode(array(10, 20)); + +-- posexplode_outer in a SELECT +SELECT posexplode_outer(array(10, 20)); + +-- posexplode in a LATERAL VIEW clause +SELECT + test.a, + test.b +FROM test + LATERAL VIEW posexplode(array(10, 20)) AS c1; + +-- posexplode_outer in a LATERAL VIEW clause +SELECT + test.a, + test.b +FROM test + LATERAL VIEW posexplode_outer(array(10, 20)) AS c1; + +-- stack in a SELECT +SELECT stack(2, 1, 2, 3); + +-- stack in a LATERAL VIEW clause +SELECT + test.a, + test.b +FROM test + LATERAL VIEW stack(2, 1, 2, 3) AS c1, c2; + +-- json_tuple in a SELECT +SELECT json_tuple('{"a":1, "b":2}', 'a', 'b'); + +-- json_tuple in a LATERAL VIEW clause +SELECT + test.a, + test.b +FROM test + LATERAL VIEW json_tuple('{"a":1, "b":2}', 'a', 'b') AS c1, c2; + +-- parse_url in a SELECT +SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST'); + +-- parse_url in a LATERAL VIEW clause +SELECT + test.a, + test.b +FROM test + LATERAL VIEW parse_url( + 'http://spark.apache.org/path?query=1', 'HOST' + ) AS c1; diff --git a/test/fixtures/dialects/spark3/select_lateral_view_supported_tvf.yml b/test/fixtures/dialects/spark3/select_lateral_view_supported_tvf.yml new file mode 100644 index 00000000000..5851a8c037e --- /dev/null +++ b/test/fixtures/dialects/spark3/select_lateral_view_supported_tvf.yml @@ -0,0 +1,717 @@ +# YML test files are auto-generated from SQL files and should not be edited by +# hand. To help enforce this, the "hash" field in the file must match a hash +# computed by SQLFluff when running the tests. Please run +# `python test/generate_parse_fixture_yml.py` to generate them after adding or +# altering SQL files. +_hash: c62157de7a615d120d66f334b4b2cc2f37f08103834733d41f8ee422e6b71004 +file: +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + function: + function_name: + function_name_identifier: explode + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + literal: '10' + - comma: ',' + - expression: + literal: '20' + - end_bracket: ) + end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + function: + function_name: + function_name_identifier: explode_outer + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + literal: '10' + - comma: ',' + - expression: + literal: '20' + - end_bracket: ) + end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + - keyword: SELECT + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: a + - comma: ',' + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: b + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: test + lateral_view_clause: + - keyword: LATERAL + - keyword: VIEW + - function: + function_name: + function_name_identifier: explode + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + literal: '3' + - comma: ',' + - expression: + literal: '4' + - end_bracket: ) + end_bracket: ) + - keyword: AS + - identifier: c2 +- statement_terminator: ; +- statement: + select_statement: + select_clause: + - keyword: SELECT + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: a + - comma: ',' + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: b + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: test + lateral_view_clause: + - keyword: LATERAL + - keyword: VIEW + - function: + function_name: + function_name_identifier: explode_outer + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + literal: '3' + - comma: ',' + - expression: + literal: '4' + - end_bracket: ) + end_bracket: ) + - keyword: AS + - identifier: c2 +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + function: + function_name: + function_name_identifier: inline + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + function: + function_name: + function_name_identifier: struct + bracketed: + - start_bracket: ( + - expression: + literal: '1' + - comma: ',' + - expression: + literal: "'a'" + - end_bracket: ) + - comma: ',' + - expression: + function: + function_name: + function_name_identifier: struct + bracketed: + - start_bracket: ( + - expression: + literal: '2' + - comma: ',' + - expression: + literal: "'b'" + - end_bracket: ) + - end_bracket: ) + end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + function: + function_name: + function_name_identifier: inline_outer + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + function: + function_name: + function_name_identifier: struct + bracketed: + - start_bracket: ( + - expression: + literal: '1' + - comma: ',' + - expression: + literal: "'a'" + - end_bracket: ) + - comma: ',' + - expression: + function: + function_name: + function_name_identifier: struct + bracketed: + - start_bracket: ( + - expression: + literal: '2' + - comma: ',' + - expression: + literal: "'b'" + - end_bracket: ) + - end_bracket: ) + end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + - keyword: SELECT + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: a + - comma: ',' + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: b + from_clause: + - keyword: FROM + - from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: test + lateral_view_clause: + - keyword: LATERAL + - keyword: VIEW + - function: + function_name: + function_name_identifier: inline + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + function: + function_name: + function_name_identifier: struct + bracketed: + - start_bracket: ( + - expression: + literal: '1' + - comma: ',' + - expression: + literal: "'a'" + - end_bracket: ) + - comma: ',' + - expression: + function: + function_name: + function_name_identifier: struct + bracketed: + - start_bracket: ( + - expression: + literal: '2' + - comma: ',' + - expression: + literal: "'b'" + - end_bracket: ) + - end_bracket: ) + end_bracket: ) + - keyword: AS + - identifier: c1 + - comma: ',' + - from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: c2 +- statement_terminator: ; +- statement: + select_statement: + select_clause: + - keyword: SELECT + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: a + - comma: ',' + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: b + from_clause: + - keyword: FROM + - from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: test + lateral_view_clause: + - keyword: LATERAL + - keyword: VIEW + - function: + function_name: + function_name_identifier: inline_outer + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + function: + function_name: + function_name_identifier: struct + bracketed: + - start_bracket: ( + - expression: + literal: '1' + - comma: ',' + - expression: + literal: "'a'" + - end_bracket: ) + - comma: ',' + - expression: + function: + function_name: + function_name_identifier: struct + bracketed: + - start_bracket: ( + - expression: + literal: '2' + - comma: ',' + - expression: + literal: "'b'" + - end_bracket: ) + - end_bracket: ) + end_bracket: ) + - keyword: AS + - identifier: c1 + - comma: ',' + - from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: c2 +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + function: + function_name: + function_name_identifier: posexplode + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + literal: '10' + - comma: ',' + - expression: + literal: '20' + - end_bracket: ) + end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + function: + function_name: + function_name_identifier: posexplode_outer + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + literal: '10' + - comma: ',' + - expression: + literal: '20' + - end_bracket: ) + end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + - keyword: SELECT + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: a + - comma: ',' + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: b + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: test + lateral_view_clause: + - keyword: LATERAL + - keyword: VIEW + - function: + function_name: + function_name_identifier: posexplode + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + literal: '10' + - comma: ',' + - expression: + literal: '20' + - end_bracket: ) + end_bracket: ) + - keyword: AS + - identifier: c1 +- statement_terminator: ; +- statement: + select_statement: + select_clause: + - keyword: SELECT + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: a + - comma: ',' + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: b + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: test + lateral_view_clause: + - keyword: LATERAL + - keyword: VIEW + - function: + function_name: + function_name_identifier: posexplode_outer + bracketed: + start_bracket: ( + expression: + function: + function_name: + function_name_identifier: array + bracketed: + - start_bracket: ( + - expression: + literal: '10' + - comma: ',' + - expression: + literal: '20' + - end_bracket: ) + end_bracket: ) + - keyword: AS + - identifier: c1 +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + function: + function_name: + function_name_identifier: stack + bracketed: + - start_bracket: ( + - expression: + literal: '2' + - comma: ',' + - expression: + literal: '1' + - comma: ',' + - expression: + literal: '2' + - comma: ',' + - expression: + literal: '3' + - end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + - keyword: SELECT + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: a + - comma: ',' + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: b + from_clause: + - keyword: FROM + - from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: test + lateral_view_clause: + - keyword: LATERAL + - keyword: VIEW + - function: + function_name: + function_name_identifier: stack + bracketed: + - start_bracket: ( + - expression: + literal: '2' + - comma: ',' + - expression: + literal: '1' + - comma: ',' + - expression: + literal: '2' + - comma: ',' + - expression: + literal: '3' + - end_bracket: ) + - keyword: AS + - identifier: c1 + - comma: ',' + - from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: c2 +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + function: + function_name: + function_name_identifier: json_tuple + bracketed: + - start_bracket: ( + - expression: + literal: "'{\"a\":1, \"b\":2}'" + - comma: ',' + - expression: + literal: "'a'" + - comma: ',' + - expression: + literal: "'b'" + - end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + - keyword: SELECT + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: a + - comma: ',' + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: b + from_clause: + - keyword: FROM + - from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: test + lateral_view_clause: + - keyword: LATERAL + - keyword: VIEW + - function: + function_name: + function_name_identifier: json_tuple + bracketed: + - start_bracket: ( + - expression: + literal: "'{\"a\":1, \"b\":2}'" + - comma: ',' + - expression: + literal: "'a'" + - comma: ',' + - expression: + literal: "'b'" + - end_bracket: ) + - keyword: AS + - identifier: c1 + - comma: ',' + - from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: c2 +- statement_terminator: ; +- statement: + select_statement: + select_clause: + keyword: SELECT + select_clause_element: + function: + function_name: + function_name_identifier: parse_url + bracketed: + - start_bracket: ( + - expression: + literal: "'http://spark.apache.org/path?query=1'" + - comma: ',' + - expression: + literal: "'HOST'" + - end_bracket: ) +- statement_terminator: ; +- statement: + select_statement: + select_clause: + - keyword: SELECT + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: a + - comma: ',' + - select_clause_element: + column_reference: + - identifier: test + - dot: . + - identifier: b + from_clause: + keyword: FROM + from_expression: + from_expression_element: + table_expression: + table_reference: + identifier: test + lateral_view_clause: + - keyword: LATERAL + - keyword: VIEW + - function: + function_name: + function_name_identifier: parse_url + bracketed: + - start_bracket: ( + - expression: + literal: "'http://spark.apache.org/path?query=1'" + - comma: ',' + - expression: + literal: "'HOST'" + - end_bracket: ) + - keyword: AS + - identifier: c1 +- statement_terminator: ;