Object Literals (#3620)

* Modify .simple() to track loops in grammars * sort dependencies of BaseExpressionElementGrammar * rationalise literals in ansi * knockon changes in athena * bigquery implications * hive knockon * postgres knockon * simplify tsql dependency * odd change to teradata? * make the object literal segment less flat. * mypy * re-add datatypes in Expression D * readjust expression * Dialect Crumbs * More mypy * Revert one of the changes with better comment * update yamls Co-authored-by: Barry Hart <barrywhart@yahoo.com>
sqlfluff · Jul 17, 2022 · 6b31de4 · 6b31de4
1 parent d83ab36
commit 6b31de4
Show file tree

Hide file tree

Showing 26 changed files with 453 additions and 552 deletions.
diff --git a/src/sqlfluff/dialects/dialect_ansi.py b/src/sqlfluff/dialects/dialect_ansi.py
@@ -416,6 +416,8 @@
         # can otherwise be easily mistaken for an identifier.
         Ref("NullLiteralSegment"),
         Ref("DateTimeLiteralGrammar"),
+        Ref("ArrayLiteralSegment"),
+        Ref("ObjectLiteralSegment"),
     ),
     AndOperatorGrammar=StringParser("AND", KeywordSegment, type="binary_operator"),
     OrOperatorGrammar=StringParser("OR", KeywordSegment, type="binary_operator"),
@@ -526,13 +528,19 @@
     ),
     # This is a placeholder for other dialects.
     SimpleArrayTypeGrammar=Nothing(),
+    # Base Expression element is the right thing to reference for everything
+    # which functions as an expression, but could include literals.
     BaseExpressionElementGrammar=OneOf(
         Ref("LiteralGrammar"),
         Ref("BareFunctionSegment"),
         Ref("IntervalExpressionSegment"),
         Ref("FunctionSegment"),
         Ref("ColumnReferenceSegment"),
         Ref("ExpressionSegment"),
+        Sequence(
+            Ref("DatatypeSegment"),
+            Ref("LiteralGrammar"),
+        ),
     ),
     FilterClauseGrammar=Sequence(
         "FILTER", Bracketed(Sequence("WHERE", Ref("ExpressionSegment")))
@@ -637,9 +645,36 @@ class ArrayLiteralSegment(BaseSegment):
     """An array literal segment."""
 
     type = "array_literal"
+    match_grammar: Matchable = Sequence(
+        Ref("SimpleArrayTypeGrammar", optional=True),
+        Bracketed(
+            Delimited(Ref("BaseExpressionElementGrammar"), optional=True),
+            bracket_type="square",
+        ),
+    )
+
+
+class ObjectLiteralSegment(BaseSegment):
+    """An object literal segment."""
+
+    type = "object_literal"
     match_grammar: Matchable = Bracketed(
-        Delimited(Ref("ExpressionSegment"), optional=True),
-        bracket_type="square",
+        Delimited(
+            Ref("ObjectLiteralElementSegment"),
+            optional=True,
+        ),
+        bracket_type="curly",
+    )
+
+
+class ObjectLiteralElementSegment(BaseSegment):
+    """An object literal element segment."""
+
+    type = "object_literal_element"
+    match_grammar: Matchable = Sequence(
+        Ref("QuotedLiteralSegment"),
+        Ref("ColonSegment"),
+        Ref("BaseExpressionElementGrammar"),
     )
 
 
@@ -1803,15 +1838,16 @@ class CaseExpressionSegment(BaseSegment):
             Sequence(
                 Ref("SingleIdentifierGrammar"), Ref("DotSegment"), Ref("StarSegment")
             ),
-            Sequence(
-                Ref("SimpleArrayTypeGrammar", optional=True), Ref("ArrayLiteralSegment")
-            ),
             Sequence(
                 Ref("StructTypeSegment"),
                 Bracketed(Delimited(Ref("ExpressionSegment"))),
             ),
             Sequence(
                 Ref("DatatypeSegment"),
+                # Don't use the full LiteralGrammar here
+                # because only some of them are applicable.
+                # Notably we shouldn't use QualifiedNumericLiteralSegment
+                # here because it looks like an arithmetic operation.
                 OneOf(
                     Ref("QuotedLiteralSegment"),
                     Ref("NumericLiteralSegment"),

diff --git a/src/sqlfluff/dialects/dialect_athena.py b/src/sqlfluff/dialects/dialect_athena.py
@@ -174,17 +174,6 @@
         NamedParser("double_quote", CodeSegment, name="quoted_literal", type="literal"),
         NamedParser("back_quote", CodeSegment, name="quoted_literal", type="literal"),
     ),
-    LiteralGrammar=OneOf(
-        Ref("QuotedLiteralSegment"),
-        Ref("NumericLiteralSegment"),
-        Ref("BooleanLiteralGrammar"),
-        Ref("QualifiedNumericLiteralSegment"),
-        # NB: Null is included in the literals, because it is a keyword which
-        # can otherwise be easily mistaken for an identifier.
-        Ref("NullLiteralSegment"),
-        Ref("DateTimeLiteralGrammar"),
-        Sequence(Ref("SimpleArrayTypeGrammar"), Ref("ArrayLiteralSegment")),
-    ),
     SimpleArrayTypeGrammar=Ref.keyword("ARRAY"),
     TrimParametersGrammar=Nothing(),
     SingleIdentifierGrammar=ansi_dialect.get_grammar("SingleIdentifierGrammar").copy(

diff --git a/src/sqlfluff/dialects/dialect_hive.py b/src/sqlfluff/dialects/dialect_hive.py
@@ -156,17 +156,6 @@
         NamedParser("double_quote", CodeSegment, name="quoted_literal", type="literal"),
         NamedParser("back_quote", CodeSegment, name="quoted_literal", type="literal"),
     ),
-    LiteralGrammar=OneOf(
-        Ref("QuotedLiteralSegment"),
-        Ref("NumericLiteralSegment"),
-        Ref("BooleanLiteralGrammar"),
-        Ref("QualifiedNumericLiteralSegment"),
-        # NB: Null is included in the literals, because it is a keyword which
-        # can otherwise be easily mistaken for an identifier.
-        Ref("NullLiteralSegment"),
-        Ref("DateTimeLiteralGrammar"),
-        Sequence(Ref("SimpleArrayTypeGrammar"), Ref("ArrayLiteralSegment")),
-    ),
     SimpleArrayTypeGrammar=Ref.keyword("ARRAY"),
     TrimParametersGrammar=Nothing(),
     SingleIdentifierGrammar=ansi_dialect.get_grammar("SingleIdentifierGrammar").copy(

diff --git a/src/sqlfluff/dialects/dialect_postgres.py b/src/sqlfluff/dialects/dialect_postgres.py
@@ -400,17 +400,11 @@
         # SelectStatementSegment so that it sits in the right
         # place corresponding to the whitespace.
     ),
-    LiteralGrammar=OneOf(
-        Ref("QuotedLiteralSegment"),
-        Ref("NumericLiteralSegment"),
-        Ref("BooleanLiteralGrammar"),
-        Ref("QualifiedNumericLiteralSegment"),
-        # NB: Null is included in the literals, because it is a keyword which
-        # can otherwise be easily mistaken for an identifier.
-        Ref("NullLiteralSegment"),
-        Ref("DateTimeLiteralGrammar"),
-        Ref("PsqlVariableGrammar"),
-        Sequence(Ref("SimpleArrayTypeGrammar"), Ref("ArrayLiteralSegment")),
+    LiteralGrammar=ansi_dialect.get_grammar("LiteralGrammar").copy(
+        insert=[
+            Ref("PsqlVariableGrammar"),
+        ],
+        before=Ref("ArrayLiteralSegment"),
     ),
     SimpleArrayTypeGrammar=Ref.keyword("ARRAY"),
     WhereClauseTerminatorGrammar=OneOf(

diff --git a/src/sqlfluff/dialects/dialect_snowflake.py b/src/sqlfluff/dialects/dialect_snowflake.py
@@ -565,16 +565,15 @@
         optional=True,
     ),
     TemporaryTransientGrammar=OneOf(Ref("TemporaryGrammar"), "TRANSIENT"),
-    BaseExpressionElementGrammar=OneOf(
-        # Allow use of CONNECT_BY_ROOT pseudo-columns.
-        # https://docs.snowflake.com/en/sql-reference/constructs/connect-by.html#:~:text=Snowflake%20supports%20the%20CONNECT_BY_ROOT,the%20Examples%20section%20below.
-        Sequence("CONNECT_BY_ROOT", Ref("ColumnReferenceSegment")),
-        Ref("LiteralGrammar"),
-        Ref("BareFunctionSegment"),
-        Ref("IntervalExpressionSegment"),
-        Ref("FunctionSegment"),
-        Ref("ColumnReferenceSegment"),
-        Ref("ExpressionSegment"),
+    BaseExpressionElementGrammar=ansi_dialect.get_grammar(
+        "BaseExpressionElementGrammar"
+    ).copy(
+        insert=[
+            # Allow use of CONNECT_BY_ROOT pseudo-columns.
+            # https://docs.snowflake.com/en/sql-reference/constructs/connect-by.html#:~:text=Snowflake%20supports%20the%20CONNECT_BY_ROOT,the%20Examples%20section%20below.
+            Sequence("CONNECT_BY_ROOT", Ref("ColumnReferenceSegment")),
+        ],
+        before=Ref("LiteralGrammar"),
     ),
     QuotedLiteralSegment=OneOf(
         # https://docs.snowflake.com/en/sql-reference/data-types-text.html#string-constants

diff --git a/src/sqlfluff/dialects/dialect_tsql.py b/src/sqlfluff/dialects/dialect_tsql.py
@@ -233,12 +233,12 @@
         )
     ),
     # Overring ANSI BaseExpressionElement to remove Interval Expression Segment
-    BaseExpressionElementGrammar=OneOf(
-        Ref("LiteralGrammar"),
-        Ref("BareFunctionSegment"),
-        Ref("FunctionSegment"),
-        Ref("ColumnReferenceSegment"),
-        Ref("ExpressionSegment"),
+    BaseExpressionElementGrammar=ansi_dialect.get_grammar(
+        "BaseExpressionElementGrammar"
+    ).copy(
+        remove=[
+            Ref("IntervalExpressionSegment"),
+        ]
     ),
     SingleIdentifierGrammar=OneOf(
         Ref("NakedIdentifierSegment"),
@@ -248,18 +248,22 @@
         Ref("ParameterNameSegment"),
         Ref("VariableIdentifierSegment"),
     ),
-    LiteralGrammar=OneOf(
-        Ref("QuotedLiteralSegment"),
-        Ref("QuotedLiteralSegmentWithN"),
-        Ref("NumericLiteralSegment"),
-        Ref("BooleanLiteralGrammar"),
-        Ref("QualifiedNumericLiteralSegment"),
-        # NB: Null is included in the literals, because it is a keyword which
-        # can otherwise be easily mistaken for an identifier.
-        Ref("NullLiteralSegment"),
-        Ref("DateTimeLiteralGrammar"),
-        Ref("ParameterNameSegment"),
-        Ref("SystemVariableSegment"),
+    LiteralGrammar=ansi_dialect.get_grammar("LiteralGrammar")
+    .copy(
+        insert=[
+            Ref("QuotedLiteralSegmentWithN"),
+        ],
+        before=Ref("NumericLiteralSegment"),
+        remove=[
+            Ref("ArrayLiteralSegment"),
+            Ref("ObjectLiteralSegment"),
+        ],
+    )
+    .copy(
+        insert=[
+            Ref("ParameterNameSegment"),
+            Ref("SystemVariableSegment"),
+        ],
     ),
     ParameterNameSegment=RegexParser(
         r"@[A-Za-z0-9_]+", CodeSegment, name="parameter", type="parameter"

diff --git a/test/fixtures/dialects/ansi/create_model_options.yml b/test/fixtures/dialects/ansi/create_model_options.yml
@@ -3,7 +3,7 @@
 # computed by SQLFluff when running the tests. Please run
 # `python test/generate_parse_fixture_yml.py`  to generate them after adding or
 # altering SQL files.
-_hash: d465ab4126507916404c690be63c85ad070a15c44102c42472463dd923179ae4
+_hash: 69affa6320f31f6d791dd891df18d54cf8d2595eacf05ec108b6f6e8cf539838
 file:
   statement:
     create_model_statement:
@@ -29,9 +29,10 @@ file:
       - parameter: INPUT_LABEL_COLS
       - comparison_operator:
           raw_comparison_operator: '='
-      - start_square_bracket: '['
-      - literal: "'label_str'"
-      - end_square_bracket: ']'
+      - array_literal:
+          start_square_bracket: '['
+          literal: "'label_str'"
+          end_square_bracket: ']'
       - end_bracket: )
     - keyword: AS
     - select_statement:

diff --git a/test/fixtures/dialects/ansi/select_fn_square_bracket_array_parameter.yml b/test/fixtures/dialects/ansi/select_fn_square_bracket_array_parameter.yml
@@ -3,7 +3,7 @@
 # computed by SQLFluff when running the tests. Please run
 # `python test/generate_parse_fixture_yml.py`  to generate them after adding or
 # altering SQL files.
-_hash: 4d6fcc39eca2aefe3fd75a046355d021e7cfd6b65a2537d786cbd4683033a697
+_hash: 4e43438e41a5797ea1f41d13fb2e7c6b63ea29e8a693ea4ae7d6b526c46e3301
 file:
   statement:
     select_statement:
@@ -18,13 +18,10 @@ file:
               expression:
                 array_literal:
                 - start_square_bracket: '['
-                - expression:
-                    literal: '1'
+                - literal: '1'
                 - comma: ','
-                - expression:
-                    literal: '2'
+                - literal: '2'
                 - comma: ','
-                - expression:
-                    literal: '3'
+                - literal: '3'
                 - end_square_bracket: ']'
               end_bracket: )
diff --git a/test/fixtures/dialects/ansi/table_expression.yml b/test/fixtures/dialects/ansi/table_expression.yml
@@ -3,7 +3,7 @@
 # computed by SQLFluff when running the tests. Please run
 # `python test/generate_parse_fixture_yml.py`  to generate them after adding or
 # altering SQL files.
-_hash: 65dd770b0d99c89c8bbbf0f377159e6cabc5a3e81052f840a68b2a53280bdc80
+_hash: 3a16a223e5ce9bfac7c1ce656591163c2f27a994b59d4b360754651b8629f315
 file:
 - statement:
     select_statement:
@@ -66,14 +66,11 @@ file:
                   expression:
                     array_literal:
                     - start_square_bracket: '['
-                    - expression:
-                        literal: '1'
+                    - literal: '1'
                     - comma: ','
-                    - expression:
-                        literal: '2'
+                    - literal: '2'
                     - comma: ','
-                    - expression:
-                        literal: '3'
+                    - literal: '3'
                     - end_square_bracket: ']'
                   end_bracket: )
           - alias_expression:
@@ -94,14 +91,11 @@ file:
                   expression:
                     array_literal:
                     - start_square_bracket: '['
-                    - expression:
-                        literal: "'a'"
+                    - literal: "'a'"
                     - comma: ','
-                    - expression:
-                        literal: "'b'"
+                    - literal: "'b'"
                     - comma: ','
-                    - expression:
-                        literal: "'c'"
+                    - literal: "'c'"
                     - end_square_bracket: ']'
                   end_bracket: )
           - alias_expression:

diff --git a/test/fixtures/dialects/athena/create_table_as_select.yml b/test/fixtures/dialects/athena/create_table_as_select.yml
@@ -3,7 +3,7 @@
 # computed by SQLFluff when running the tests. Please run
 # `python test/generate_parse_fixture_yml.py`  to generate them after adding or
 # altering SQL files.
-_hash: e16771ae3742a10d3a72e1eccff5bcc2c153477899638499ea18a319d760ca12
+_hash: 6469816f4e4c64b9af86701ef4f034af0635749a38b71dfe230ee7ef1c3a899c
 file:
   statement:
     create_table_statement:
@@ -27,11 +27,10 @@ file:
       - keyword: partitioned_by
       - comparison_operator:
           raw_comparison_operator: '='
-      - keyword: array
       - array_literal:
+          keyword: array
           start_square_bracket: '['
-          expression:
-            literal: "'load_date'"
+          literal: "'load_date'"
           end_square_bracket: ']'
       - end_bracket: )
     - keyword: AS