Merge branch 'main' into sqlfluffignore-no-glob

.github/workflows/ci-tests.yml

@@ -112,7 +112,7 @@ jobs:
     strategy:
       matrix:
         dbt-version: [ 'dbt018', 'dbt019', 'dbt020', 'dbt021' ]
-    name: DBT ${{ matrix.dbt-version }} tests
+    name: DBT Plugin ${{ matrix.dbt-version }} tests
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
@@ -124,7 +124,7 @@ jobs:
         pip install tox
     - name: Run the tests
       run: |
-        tox -e ${{ matrix.dbt-version }} -- -m dbt
+        tox -e ${{ matrix.dbt-version }} -- plugins/sqlfluff-templater-dbt
     - name: Upload Coverage Report
       uses: codecov/codecov-action@v1
       with:
@@ -164,7 +164,7 @@ jobs:
         files: ./coverage.xml
   python-windows-dbt-tests:
     runs-on: windows-latest
-    name: Python 3.8 Windows dbt tests
+    name: DBT Plugin Python 3.8 Windows tests
     steps:
     - name: Set git to use LF
       run: |
@@ -184,7 +184,7 @@ jobs:
       # Do not set explicitly temp dir for dbt as causes problems
       # None of these test need temp dir set
       run: |
-        python -m tox -e dbt018-winpy -- -m dbt
+        python -m tox -e dbt018-winpy -- plugins/sqlfluff-templater-dbt
     - name: Upload Coverage Report
       uses: codecov/codecov-action@v1
       with:

CONTRIBUTING.md

@@ -155,7 +155,7 @@ pytest -k L012 -v
 To run the dbt-related tests you will have to explicitly include these tests:
 
 ```shell
-tox -e cov-init,dbt018-py38,cov-report-dbt -- -m "dbt"
+tox -e cov-init,dbt018-py38,cov-report-dbt -- plugins/sqlfluff-templater-dbt
 ```
 
 For more information on adding and running test cases see the [Parser Test README](test/fixtures/dialects/README.md) and the [Rules Test README](test/fixtures/rules/std_rule_cases/README.md).

plugins/sqlfluff-templater-dbt/test/__init__.py

@@ -0,0 +1 @@
+"""Init PY for tests."""

test/fixtures/dbt/templater.py → plugins/sqlfluff-templater-dbt/test/fixtures/dbt/templater.py

@@ -11,8 +11,8 @@
     },
     "templater": {
         "dbt": {
-            "profiles_dir": "test/fixtures/dbt",
-            "project_dir": "test/fixtures/dbt/dbt_project",
+            "profiles_dir": "plugins/sqlfluff-templater-dbt/test/fixtures/dbt",
+            "project_dir": "plugins/sqlfluff-templater-dbt/test/fixtures/dbt/dbt_project",
         },
     },
 }

plugins/sqlfluff-templater-dbt/test/linter_test.py

@@ -0,0 +1,24 @@
+"""The Test file for the linter class."""
+
+import os
+import os.path
+
+from sqlfluff.core import Linter, FluffConfig
+from test.fixtures.dbt.templater import DBT_FLUFF_CONFIG, project_dir  # noqa: F401
+
+
+def test__linter__skip_dbt_model_disabled(project_dir):  # noqa
+    """Test that the linter skips disabled dbt models."""
+    conf = FluffConfig(configs=DBT_FLUFF_CONFIG)
+    lntr = Linter(config=conf)
+    model_file_path = os.path.join(
+        project_dir, "models/my_new_project/disabled_model.sql"
+    )
+    linted_path = lntr.lint_path(path=model_file_path)
+    # Check that the file is still there
+    assert len(linted_path.files) == 1
+    linted_file = linted_path.files[0]
+    # Normalise paths to control for OS variance
+    assert os.path.normpath(linted_file.path) == os.path.normpath(model_file_path)
+    assert not linted_file.templated_file
+    assert not linted_file.tree

plugins/sqlfluff-templater-dbt/test/rules_test.py

@@ -0,0 +1,29 @@
+"""Tests for the standard set of rules."""
+import pytest
+import os
+
+from sqlfluff.core.config import FluffConfig
+from sqlfluff.testing.rules import assert_rule_raises_violations_in_file
+
+from test.fixtures.dbt.templater import (  # noqa
+    DBT_FLUFF_CONFIG,
+    project_dir,
+    dbt_templater,
+)
+
+
+@pytest.mark.parametrize(
+    "rule,path,violations",
+    [
+        # Group By
+        ("L021", "models/my_new_project/select_distinct_group_by.sql", [(1, 8)]),
+    ],
+)
+def test__rules__std_file_dbt(rule, path, violations, project_dir):  # noqa
+    """Test the linter finds the given errors in (and only in) the right places (DBT)."""
+    assert_rule_raises_violations_in_file(
+        rule=rule,
+        fpath=os.path.join(project_dir, path),
+        violations=violations,
+        fluff_config=FluffConfig(configs=DBT_FLUFF_CONFIG, overrides=dict(rules=rule)),
+    )

test/core/templaters/dbt_test.py → plugins/sqlfluff-templater-dbt/test/templater_test.py

@@ -14,7 +14,6 @@
 )
 
 
-@pytest.mark.dbt
 def test__templater_dbt_missing(dbt_templater, project_dir):  # noqa: F811
     """Check that a nice error is returned when dbt module is missing."""
     try:
@@ -32,7 +31,6 @@ def test__templater_dbt_missing(dbt_templater, project_dir):  # noqa: F811
         )
 
 
-@pytest.mark.dbt
 def test__templater_dbt_profiles_dir_expanded(dbt_templater):  # noqa: F811
     """Check that the profiles_dir is expanded."""
     dbt_templater.sqlfluff_config = FluffConfig(
@@ -58,7 +56,6 @@ def test__templater_dbt_profiles_dir_expanded(dbt_templater):  # noqa: F811
         "use_var.sql",
     ],
 )
-@pytest.mark.dbt
 def test__templater_dbt_templating_result(
     project_dir, dbt_templater, fname  # noqa: F811
 ):
@@ -68,7 +65,10 @@ def test__templater_dbt_templating_result(
         fname=os.path.join(project_dir, "models/my_new_project/", fname),
         config=FluffConfig(configs=DBT_FLUFF_CONFIG),
     )
-    assert str(templated_file) == open("test/fixtures/dbt/" + fname).read()
+    assert (
+        str(templated_file)
+        == open("plugins/sqlfluff-templater-dbt/test/fixtures/dbt/" + fname).read()
+    )
 
 
 @pytest.mark.parametrize(
@@ -108,7 +108,6 @@ def test__templater_dbt_templating_result(
         ],
     ],
 )
-@pytest.mark.dbt
 def test__templater_dbt_sequence_files_ephemeral_dependency(
     project_dir, dbt_templater, fnames_input, fnames_expected_sequence  # noqa: F811
 ):
@@ -122,7 +121,6 @@ def test__templater_dbt_sequence_files_ephemeral_dependency(
     assert list(result) == expected
 
 
-@pytest.mark.dbt
 @pytest.mark.parametrize(
     "raw_file,templated_file,result",
     [
@@ -160,7 +158,6 @@ def test__templater_dbt_slice_file_wrapped_test(
         "models/my_new_project/multiple_trailing_newline.sql",
     ],
 )
-@pytest.mark.dbt
 def test__templater_dbt_templating_test_lex(
     project_dir, dbt_templater, fname  # noqa: F811
 ):
@@ -186,7 +183,6 @@ def test__templater_dbt_templating_test_lex(
     )
 
 
-@pytest.mark.dbt
 def test__templater_dbt_skips_disabled_model(dbt_templater, project_dir):  # noqa: F811
     """A disabled dbt model should be skipped."""
     with pytest.raises(SQLTemplaterSkipFile, match=r"model was disabled"):
@@ -207,7 +203,6 @@ def test__templater_dbt_skips_disabled_model(dbt_templater, project_dir):  # noq
         "L034_test.sql",
     ],
 )
-@pytest.mark.dbt
 def test__dbt_templated_models_do_not_raise_lint_error(
     project_dir, fname  # noqa: F811
 ):
@@ -220,7 +215,6 @@ def test__dbt_templated_models_do_not_raise_lint_error(
     assert len(violations) == 0
 
 
-@pytest.mark.dbt
 def test__templater_dbt_templating_absolute_path(
     project_dir, dbt_templater  # noqa: F811
 ):
@@ -248,14 +242,13 @@ def test__templater_dbt_templating_absolute_path(
         ("exception_connect_database.sql", "dbt tried to connect to the database"),
     ],
 )
-@pytest.mark.dbt
 def test__templater_dbt_handle_exceptions(
     project_dir, dbt_templater, fname, exception_msg  # noqa: F811
 ):
     """Test that exceptions during compilation are returned as violation."""
     from dbt.adapters.factory import get_adapter
 
-    src_fpath = "test/fixtures/dbt/error_models/" + fname
+    src_fpath = "plugins/sqlfluff-templater-dbt/test/fixtures/dbt/error_models/" + fname
     target_fpath = os.path.abspath(
         os.path.join(project_dir, "models/my_new_project/", fname)
     )
@@ -276,7 +269,6 @@ def test__templater_dbt_handle_exceptions(
     assert violations[0].desc().replace("\\", "/").startswith(exception_msg)
 
 
-@pytest.mark.dbt
 def test__project_dir_does_not_exist_error(dbt_templater, caplog):  # noqa: F811
     """Test that an error is logged if the specified dbt project directory doesn't exist."""
     dbt_templater.sqlfluff_config = FluffConfig(

src/sqlfluff/dialects/dialect_tsql.py

@@ -17,7 +17,6 @@
     Delimited,
     Matchable,
     NamedParser,
-    StartsWith,
     OptionallyBracketed,
     Dedent,
     BaseFileSegment,
@@ -428,21 +427,22 @@ class DeclareStatementSegment(BaseSegment):
     match_grammar = Sequence(
         "DECLARE",
         Ref("ParameterNameSegment"),
-        AnyNumberOf(
-            Ref("ParameterNameSegment"),
-            Ref("CommaSegment", optional=True),
-        ),
         Ref("DatatypeSegment"),
         Sequence(
             Ref("EqualsSegment"),
-            OneOf(
-                Ref("LiteralGrammar"),
-                Bracketed(Ref("SelectStatementSegment")),
-                Ref("BareFunctionSegment"),
-                Ref("FunctionSegment"),
-            ),
+            Ref("ExpressionSegment"),
             optional=True,
         ),
+        AnyNumberOf(
+            Ref("CommaSegment"),
+            Ref("ParameterNameSegment"),
+            Ref("DatatypeSegment"),
+            Sequence(
+                Ref("EqualsSegment"),
+                Ref("ExpressionSegment"),
+                optional=True,
+            ),
+        ),
         Ref("DelimiterSegment", optional=True),
     )
 
@@ -628,11 +628,12 @@ class SetStatementSegment(BaseSegment):
 
     Setting an already declared variable or global variable.
     https://docs.microsoft.com/en-us/sql/t-sql/statements/set-statements-transact-sql?view=sql-server-ver15
+
+    https://docs.microsoft.com/en-us/sql/t-sql/language-elements/set-local-variable-transact-sql?view=sql-server-ver15
     """
 
     type = "set_segment"
-    match_grammar = StartsWith("SET")
-    parse_grammar = Sequence(
+    match_grammar = Sequence(
         "SET",
         OneOf(
             Ref("ParameterNameSegment"),
@@ -682,25 +683,7 @@ class SetStatementSegment(BaseSegment):
             "OFF",
             Sequence(
                 Ref("EqualsSegment"),
-                OneOf(
-                    Delimited(
-                        OneOf(
-                            Ref("LiteralGrammar"),
-                            Bracketed(Ref("SelectStatementSegment")),
-                            Ref("FunctionSegment"),
-                            Bracketed(
-                                Delimited(
-                                    OneOf(
-                                        Ref("LiteralGrammar"),
-                                        Bracketed(Ref("SelectStatementSegment")),
-                                        Ref("BareFunctionSegment"),
-                                        Ref("FunctionSegment"),
-                                    )
-                                )
-                            ),
-                        )
-                    )
-                ),
+                Ref("ExpressionSegment"),
             ),
         ),
     )
@@ -1166,6 +1149,7 @@ class CreateTableAsSelectStatementSegment(BaseSegment):
         Ref("TableDistributionIndexClause"),
         "AS",
         Ref("SelectableGrammar"),
+        Ref("DelimiterSegment", optional=True),
     )
 
 

src/sqlfluff/rules/L048.py

@@ -44,6 +44,6 @@ def _missing_whitespace(seg, before=True):
         https://github.com/sqlfluff/sqlfluff/issues/943
         """
         simple_res = Rule_L006._missing_whitespace(seg, before=before)
-        if not before and seg and seg.is_type("comma"):
+        if not before and seg and seg.is_type("comma", "statement_terminator"):
             return False
         return simple_res

test/core/linter_test.py

@@ -2,8 +2,6 @@
 
 import pytest
 import logging
-import os
-import os.path
 from typing import List
 from unittest.mock import patch
 
@@ -13,7 +11,6 @@
 from sqlfluff.cli.formatters import CallbackFormatter
 from sqlfluff.core.linter import LintingResult, NoQaDirective
 import sqlfluff.core.linter as linter
-from test.fixtures.dbt.templater import DBT_FLUFF_CONFIG, project_dir  # noqa: F401
 from sqlfluff.core.templaters import TemplatedFile
 
 
@@ -617,24 +614,6 @@ def test_linter_noqa_with_templating():
     assert not result.get_violations()
 
 
-@pytest.mark.dbt
-def test__linter__skip_dbt_model_disabled(project_dir):  # noqa
-    """Test that the linter skips disabled dbt models."""
-    conf = FluffConfig(configs=DBT_FLUFF_CONFIG)
-    lntr = Linter(config=conf)
-    model_file_path = os.path.join(
-        project_dir, "models/my_new_project/disabled_model.sql"
-    )
-    linted_path = lntr.lint_path(path=model_file_path)
-    # Check that the file is still there
-    assert len(linted_path.files) == 1
-    linted_file = linted_path.files[0]
-    # Normalise paths to control for OS variance
-    assert os.path.normpath(linted_file.path) == os.path.normpath(model_file_path)
-    assert not linted_file.templated_file
-    assert not linted_file.tree
-
-
 def test_delayed_exception():
     """Test that DelayedException stores and reraises a stored exception."""
     ve = ValueError()

test/core/rules/config_test.py

@@ -9,11 +9,6 @@
 from sqlfluff.core.parser import NewlineSegment
 from sqlfluff.testing.rules import get_rule_from_set
 
-from test.fixtures.dbt.templater import (  # noqa
-    DBT_FLUFF_CONFIG,
-    project_dir,
-    dbt_templater,
-)
 from test.fixtures.rules.custom.L000 import Rule_L000
 from test.fixtures.rules.custom.S000 import Rule_S000
 from sqlfluff.core.rules.loader import get_rules_from_path