Spark3: AuxiliaryFILE and JAR statements

src/sqlfluff/dialects/dialect_spark3.py

@@ -31,6 +31,7 @@
     SymbolSegment,
     Anything,
     StartsWith,
+    RegexParser,
 )
 from sqlfluff.core.parser.segments.raw import CodeSegment, KeywordSegment
 from sqlfluff.dialects.dialect_spark3_keywords import (
@@ -279,11 +280,15 @@
     EqualsSegment_b=StringParser(
         "<=>", SymbolSegment, name="equals", type="comparison_operator"
     ),
-    FileKeywordSegment=StringParser(
-        "FILE", KeywordSegment, name="file", type="file_type"
+    FileKeywordSegment=RegexParser(
+        "FILES?", KeywordSegment, name="file", type="file_keyword"
+    ),
+    JarKeywordSegment=RegexParser(
+        "JARS?", KeywordSegment, name="jar", type="file_keyword"
+    ),
+    WhlKeywordSegment=StringParser(
+        "WHL", KeywordSegment, name="whl", type="file_keyword"
     ),
-    JarKeywordSegment=StringParser("JAR", KeywordSegment, name="jar", type="file_type"),
-    WhlKeywordSegment=StringParser("WHL", KeywordSegment, name="whl", type="file_type"),
     # Add relevant Hive Grammar
     BracketedPropertyListGrammar=hive_dialect.get_grammar(
         "BracketedPropertyListGrammar"
@@ -1709,18 +1714,66 @@ class ExplainStatementSegment(BaseSegment):
 
 # Auxiliary Statements
 @spark3_dialect.segment()
-class AddExecutablePackage(BaseSegment):
-    """A `ADD JAR` statement.
+class AddFileSegment(BaseSegment):
+    """A `ADD {FILE | FILES}` statement.
+
+    https://spark.apache.org/docs/latest/sql-ref-syntax-aux-resource-mgmt-add-file.html
+    """
+
+    type = "add_file"
+
+    match_grammar = Sequence(
+        "ADD",
+        Ref("FileKeywordSegment"),
+        AnyNumberOf(Ref("QuotedLiteralSegment")),
+    )
+
+
+@spark3_dialect.segment()
+class AddJarSegment(BaseSegment):
+    """A `ADD {JAR | JARS}` statement.
 
     https://spark.apache.org/docs/latest/sql-ref-syntax-aux-resource-mgmt-add-jar.html
     """
 
-    type = "add_executable_package"
+    type = "add_jar"
 
     match_grammar = Sequence(
         "ADD",
-        Ref("ResourceFileGrammar"),
-        Ref("QuotedLiteralSegment"),
+        Ref("JarKeywordSegment"),
+        AnyNumberOf(Ref("QuotedLiteralSegment")),
+    )
+
+
+@spark3_dialect.segment()
+class ListFileSegment(BaseSegment):
+    """A `LIST {FILE | FILES}` statement.
+
+    https://spark.apache.org/docs/latest/sql-ref-syntax-aux-resource-mgmt-list-file.html
+    """
+
+    type = "list_file"
+
+    match_grammar = Sequence(
+        "LIST",
+        Ref("FileKeywordSegment"),
+        AnyNumberOf(Ref("QuotedLiteralSegment")),
+    )
+
+
+@spark3_dialect.segment()
+class ListJarSegment(BaseSegment):
+    """A `ADD {JAR | JARS}` statement.
+
+    https://spark.apache.org/docs/latest/sql-ref-syntax-aux-resource-mgmt-add-jar.html
+    """
+
+    type = "list_jar"
+
+    match_grammar = Sequence(
+        "LIST",
+        Ref("JarKeywordSegment"),
+        AnyNumberOf(Ref("QuotedLiteralSegment")),
     )
 
 
@@ -1790,7 +1843,10 @@ class StatementSegment(BaseSegment):
             Ref("MsckRepairTableStatementSegment"),
             Ref("UseDatabaseStatementSegment"),
             # Auxiliary Statements
-            Ref("AddExecutablePackage"),
+            Ref("AddFileSegment"),
+            Ref("AddJarSegment"),
+            Ref("ListFileSegment"),
+            Ref("ListJarSegment"),
             Ref("RefreshStatementSegment"),
             Ref("RefreshTableStatementSegment"),
             Ref("RefreshFunctionStatementSegment"),

test/fixtures/dialects/spark3/add_file.sql

@@ -0,0 +1,12 @@
+ADD FILE "/path/to/file/abc.txt";
+
+ADD FILE '/another/test.txt';
+
+ADD FILE "/path with space/abc.txt";
+
+ADD FILE "/path/to/some/directory";
+
+ADD FILES "/path with space/cde.txt" '/path with space/fgh.txt';
+
+-- NB: Non-quoted paths are not supported in SQLFluff currently
+--ADD FILE /tmp/test;

test/fixtures/dialects/spark3/add_file.yml

@@ -0,0 +1,38 @@
+# YML test files are auto-generated from SQL files and should not be edited by
+# hand. To help enforce this, the "hash" field in the file must match a hash
+# computed by SQLFluff when running the tests. Please run
+# `python test/generate_parse_fixture_yml.py`  to generate them after adding or
+# altering SQL files.
+_hash: b2703f9527e83c7cb69caac9fdfe16f15000f9ec93ce59c83471d05b24db927c
+file:
+- statement:
+    add_file:
+      keyword: ADD
+      file_keyword: FILE
+      literal: '"/path/to/file/abc.txt"'
+- statement_terminator: ;
+- statement:
+    add_file:
+      keyword: ADD
+      file_keyword: FILE
+      literal: "'/another/test.txt'"
+- statement_terminator: ;
+- statement:
+    add_file:
+      keyword: ADD
+      file_keyword: FILE
+      literal: '"/path with space/abc.txt"'
+- statement_terminator: ;
+- statement:
+    add_file:
+      keyword: ADD
+      file_keyword: FILE
+      literal: '"/path/to/some/directory"'
+- statement_terminator: ;
+- statement:
+    add_file:
+    - keyword: ADD
+    - file_keyword: FILES
+    - literal: '"/path with space/cde.txt"'
+    - literal: "'/path with space/fgh.txt'"
+- statement_terminator: ;

test/fixtures/dialects/spark3/add_jar.sql

@@ -0,0 +1,18 @@
+ADD JAR "/path/to/some.jar";
+
+ADD JAR '/some/other.jar';
+
+ADD JAR "/path with space/abc.jar";
+
+ADD JARS "/path with space/def.jar" '/path with space/ghi.jar';
+
+ADD JAR "ivy://group:module:version";
+
+ADD JAR "ivy://group:module:version?transitive=false";
+
+ADD JAR "ivy://group:module:version?transitive=true";
+
+ADD JAR "ivy://group:module:version?exclude=group:module&transitive=true";
+
+-- NB: Non-quoted paths are not supported in SQLFluff currently
+--ADD JAR /tmp/test.jar;

test/fixtures/dialects/spark3/add_jar.yml

@@ -0,0 +1,56 @@
+# YML test files are auto-generated from SQL files and should not be edited by
+# hand. To help enforce this, the "hash" field in the file must match a hash
+# computed by SQLFluff when running the tests. Please run
+# `python test/generate_parse_fixture_yml.py`  to generate them after adding or
+# altering SQL files.
+_hash: 8fa0c08d81721ba35840db1f3f45ab22c2ab7e76698ca1ec795e625731132792
+file:
+- statement:
+    add_jar:
+      keyword: ADD
+      file_keyword: JAR
+      literal: '"/path/to/some.jar"'
+- statement_terminator: ;
+- statement:
+    add_jar:
+      keyword: ADD
+      file_keyword: JAR
+      literal: "'/some/other.jar'"
+- statement_terminator: ;
+- statement:
+    add_jar:
+      keyword: ADD
+      file_keyword: JAR
+      literal: '"/path with space/abc.jar"'
+- statement_terminator: ;
+- statement:
+    add_jar:
+    - keyword: ADD
+    - file_keyword: JARS
+    - literal: '"/path with space/def.jar"'
+    - literal: "'/path with space/ghi.jar'"
+- statement_terminator: ;
+- statement:
+    add_jar:
+      keyword: ADD
+      file_keyword: JAR
+      literal: '"ivy://group:module:version"'
+- statement_terminator: ;
+- statement:
+    add_jar:
+      keyword: ADD
+      file_keyword: JAR
+      literal: '"ivy://group:module:version?transitive=false"'
+- statement_terminator: ;
+- statement:
+    add_jar:
+      keyword: ADD
+      file_keyword: JAR
+      literal: '"ivy://group:module:version?transitive=true"'
+- statement_terminator: ;
+- statement:
+    add_jar:
+      keyword: ADD
+      file_keyword: JAR
+      literal: '"ivy://group:module:version?exclude=group:module&transitive=true"'
+- statement_terminator: ;

test/fixtures/dialects/spark3/create_function.yml

@@ -3,7 +3,7 @@
 # computed by SQLFluff when running the tests. Please run
 # `python test/generate_parse_fixture_yml.py`  to generate them after adding or
 # altering SQL files.
-_hash: 3bee12bfaf3f18f18d67fa11abb49e22092fa8919b2209ca3eab7b042b5ef191
+_hash: cf3bd31e3fc6f6655b428beaa28713dd7ca8caf72d0f0ed27bdf6e21d1178cbd
 file:
 - statement:
     create_function_statement:
@@ -19,7 +19,7 @@ file:
     - keyword: AS
     - literal: '"class_name"'
     - keyword: USING
-    - file_type: FILE
+    - file_keyword: FILE
     - literal: '"resource_locations"'
 - statement_terminator: ;
 - statement:
@@ -30,7 +30,7 @@ file:
     - keyword: AS
     - literal: "'SimpleUdf'"
     - keyword: USING
-    - file_type: JAR
+    - file_keyword: JAR
     - literal: "'/tmp/SimpleUdf.jar'"
 - statement_terminator: ;
 - statement:
@@ -42,7 +42,7 @@ file:
     - keyword: AS
     - literal: "'SimpleUdf'"
     - keyword: USING
-    - file_type: JAR
+    - file_keyword: JAR
     - literal: "'/tmp/SimpleUdf.jar'"
 - statement_terminator: ;
 - statement:
@@ -55,7 +55,7 @@ file:
     - keyword: AS
     - literal: "'SimpleUdfR'"
     - keyword: USING
-    - file_type: JAR
+    - file_keyword: JAR
     - literal: "'/tmp/SimpleUdfR.jar'"
 - statement_terminator: ;
 - statement:

test/fixtures/dialects/spark3/create_table_hiveformat.yml

@@ -3,7 +3,7 @@
 # computed by SQLFluff when running the tests. Please run
 # `python test/generate_parse_fixture_yml.py`  to generate them after adding or
 # altering SQL files.
-_hash: c9e60e4107d8e23973b4b508706d0b2ed4d9b4d56cdcd047f8d2280ef39d5b6f
+_hash: f75c8976e83dd9f1911b0ed97169b53302402d55e118739f13ba3d117ca0fe90
 file:
 - statement:
     create_table_statement:
@@ -469,9 +469,9 @@ file:
       - end_bracket: )
 - statement_terminator: ;
 - statement:
-    add_executable_package:
+    add_jar:
       keyword: ADD
-      file_type: JAR
+      file_keyword: JAR
       literal: "'/tmp/hive_serde_example.jar'"
 - statement_terminator: ;
 - statement:

test/fixtures/dialects/spark3/explain.yml

@@ -3,7 +3,7 @@
 # computed by SQLFluff when running the tests. Please run
 # `python test/generate_parse_fixture_yml.py`  to generate them after adding or
 # altering SQL files.
-_hash: 91c3c5d1f04023d225b7917117ea6f36460026926151f116bfeb528d1ad761ed
+_hash: 75a855a4e5e9da38ec7c9e10a2e9a841ba415c0a8a25c12f756f8825696678d1
 file:
 - statement:
     explain_statement:
@@ -161,7 +161,7 @@ file:
         - keyword: AS
         - literal: '"class_name"'
         - keyword: USING
-        - file_type: FILE
+        - file_keyword: FILE
         - literal: '"resource_locations"'
 - statement_terminator: ;
 - statement:

test/fixtures/dialects/spark3/list_file.sql

@@ -0,0 +1,12 @@
+LIST FILE "/path/to/file/abc.txt";
+
+LIST FILE '/another/test.txt';
+
+LIST FILE "/path with space/abc.txt";
+
+LIST FILE "/path/to/some/directory";
+
+LIST FILES "/path with space/cde.txt" '/path with space/fgh.txt';
+
+-- NB: Non-quoted paths are not supported in SQLFluff currently
+--LIST FILE /tmp/test;

test/fixtures/dialects/spark3/list_file.yml

@@ -0,0 +1,38 @@
+# YML test files are auto-generated from SQL files and should not be edited by
+# hand. To help enforce this, the "hash" field in the file must match a hash
+# computed by SQLFluff when running the tests. Please run
+# `python test/generate_parse_fixture_yml.py`  to generate them after adding or
+# altering SQL files.
+_hash: 009aee055175380c9d1c162318ac313c90d6bb95285120c1a0ebda94ab38ceed
+file:
+- statement:
+    list_file:
+      keyword: LIST
+      file_keyword: FILE
+      literal: '"/path/to/file/abc.txt"'
+- statement_terminator: ;
+- statement:
+    list_file:
+      keyword: LIST
+      file_keyword: FILE
+      literal: "'/another/test.txt'"
+- statement_terminator: ;
+- statement:
+    list_file:
+      keyword: LIST
+      file_keyword: FILE
+      literal: '"/path with space/abc.txt"'
+- statement_terminator: ;
+- statement:
+    list_file:
+      keyword: LIST
+      file_keyword: FILE
+      literal: '"/path/to/some/directory"'
+- statement_terminator: ;
+- statement:
+    list_file:
+    - keyword: LIST
+    - file_keyword: FILES
+    - literal: '"/path with space/cde.txt"'
+    - literal: "'/path with space/fgh.txt'"
+- statement_terminator: ;

test/fixtures/dialects/spark3/list_jar.sql

@@ -0,0 +1,18 @@
+LIST JAR "/path/to/some.jar";
+
+LIST JAR '/some/other.jar';
+
+LIST JAR "/path with space/abc.jar";
+
+LIST JARS "/path with space/def.jar" '/path with space/ghi.jar';
+
+LIST JAR "ivy://group:module:version";
+
+LIST JAR "ivy://group:module:version?transitive=false";
+
+LIST JAR "ivy://group:module:version?transitive=true";
+
+LIST JAR "ivy://group:module:version?exclude=group:module&transitive=true";
+
+-- NB: Non-quoted paths are not supported in SQLFluff currently
+--LIST JAR /tmp/test.jar;