Add flyway variables support via placeholder templater (#4026)

* adding new style for supporting flyway placeholder variables in sql

* add fixture for flyway style placeholder templating

* adding placeholder test for flyway_var style

* mandate flyway_var pattern be enclosed in curly brakcets always

* adding test__config__load_placeholder_cfg

Co-authored-by: Barry Hart <barrywhart@yahoo.com>

src/sqlfluff/core/config.py

@@ -302,7 +302,7 @@ def _get_config_elems_from_file(cls, fpath: str) -> List[ConfigElemType]:
         # Disable interpolation so we can load macros
         kw: Dict = {}
         kw["interpolation"] = None
-        config = configparser.ConfigParser(**kw)
+        config = configparser.ConfigParser(delimiters="=", **kw)
         # NB: We want to be case sensitive in how we read from files,
         # because jinja is also case sensitive. To do this we override
         # the optionxform attribute.

src/sqlfluff/core/templaters/placeholder.py

@@ -35,6 +35,8 @@
     "dollar": regex.compile(
         r"(?<![:\w\x5c])\${?(?P<param_name>[\w_]+)}?", regex.UNICODE
     ),
+    # e.g. USE ${flyway:database}.schema_name;
+    "flyway_var": regex.compile(r"\${(?P<param_name>\w+[:\w_]+)}", regex.UNICODE),
     # e.g. WHERE bla = ?
     "question_mark": regex.compile(r"(?<![:\w\x5c])\?", regex.UNICODE),
     # e.g. WHERE bla = $3 or WHERE bla = ${3}

test/core/config_test.py

@@ -137,6 +137,28 @@ def test__config__load_toml():
     }
 
 
+def test__config__load_placeholder_cfg():
+    """Test loading a sqlfluff configuration file for placeholder templater."""
+    c = ConfigLoader()
+    cfg = c.load_config_file(
+        os.path.join("test", "fixtures", "config", "placeholder"),
+        ".sqlfluff-placeholder",
+    )
+    assert cfg == {
+        "core": {
+            "testing_val": "foobar",
+            "testing_int": 4,
+        },
+        "bar": {"foo": "barbar"},
+        "templater": {
+            "placeholder": {
+                "param_style": "flyway_var",
+                "flyway:database": "test_db",
+            }
+        },
+    }
+
+
 def test__config__iter_config_paths_right_order():
     """Test that config paths are fetched ordered by priority."""
     c = ConfigLoader()

test/core/templaters/placeholder_test.py

@@ -289,6 +289,22 @@ def test__templater_raw():
                 start_date="'2021-10-01'",
             ),
         ),
+        (
+            "USE ${flyway:database}.test_schema;",
+            "flyway_var",
+            "USE test_db.test_schema;",
+            {
+                "flyway:database": "test_db",
+            },
+        ),
+        (
+            "SELECT metadata$filename, $1 FROM @stg_data_export_${env_name};",
+            "flyway_var",
+            "SELECT metadata$filename, $1 FROM @stg_data_export_staging;",
+            {
+                "env_name": "staging",
+            },
+        ),
     ],
     ids=[
         "no_changes",
@@ -306,6 +322,8 @@ def test__templater_raw():
         "numeric_dollar_with_braces_and_string",
         "percent",
         "ampersand",
+        "flyway_var",
+        "flyway_var",
     ],
 )
 def test__templater_param_style(instr, expected_outstr, param_style, values):

test/fixtures/config/placeholder/.sqlfluff-placeholder

@@ -0,0 +1,10 @@
+[sqlfluff]
+testing_val=foobar
+testing_int=4
+
+[sqlfluff:bar]
+foo=barbar
+
+[sqlfluff:templater:placeholder]
+param_style = flyway_var
+flyway:database = test_db

test/fixtures/templater/placeholder_flyway_var/placeholder_flyway_var_a.sql

@@ -0,0 +1,5 @@
+USE ${flyway:database}.test_schema;
+
+CREATE OR REPLACE STAGE stg_data_export_${env_name}
+URL = 's3://${s3_data_lake_bucket}/${env_name}/exports/stg_data_export'
+STORAGE_INTEGRATION = s3_integ_main;