From 4d2a83c556c442ae73d473692cd69b29851f9425 Mon Sep 17 00:00:00 2001
From: Sreejith Madhavan <34316602+srjonemed@users.noreply.github.com>
Date: Sun, 13 Nov 2022 07:41:25 -0800
Subject: [PATCH] Add flyway variables support via placeholder templater
 (#4026)

* adding new style for supporting flyway placeholder variables in sql

* add fixture for flyway style placeholder templating

* adding placeholder test for flyway_var style

* mandate flyway_var pattern be enclosed in curly brakcets always

* adding test__config__load_placeholder_cfg

Co-authored-by: Barry Hart <barrywhart@yahoo.com>
---
 src/sqlfluff/core/config.py                   |  2 +-
 src/sqlfluff/core/templaters/placeholder.py   |  2 ++
 test/core/config_test.py                      | 22 +++++++++++++++++++
 test/core/templaters/placeholder_test.py      | 18 +++++++++++++++
 .../config/placeholder/.sqlfluff-placeholder  | 10 +++++++++
 .../placeholder_flyway_var_a.sql              |  5 +++++
 6 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 test/fixtures/config/placeholder/.sqlfluff-placeholder
 create mode 100644 test/fixtures/templater/placeholder_flyway_var/placeholder_flyway_var_a.sql

diff --git a/src/sqlfluff/core/config.py b/src/sqlfluff/core/config.py
index 1b7a5152da5..8d8ee2fa3ed 100644
--- a/src/sqlfluff/core/config.py
+++ b/src/sqlfluff/core/config.py
@@ -302,7 +302,7 @@ def _get_config_elems_from_file(cls, fpath: str) -> List[ConfigElemType]:
         # Disable interpolation so we can load macros
         kw: Dict = {}
         kw["interpolation"] = None
-        config = configparser.ConfigParser(**kw)
+        config = configparser.ConfigParser(delimiters="=", **kw)
         # NB: We want to be case sensitive in how we read from files,
         # because jinja is also case sensitive. To do this we override
         # the optionxform attribute.
diff --git a/src/sqlfluff/core/templaters/placeholder.py b/src/sqlfluff/core/templaters/placeholder.py
index 8627e4bf576..e24067a9df4 100644
--- a/src/sqlfluff/core/templaters/placeholder.py
+++ b/src/sqlfluff/core/templaters/placeholder.py
@@ -35,6 +35,8 @@
     "dollar": regex.compile(
         r"(?<![:\w\x5c])\${?(?P<param_name>[\w_]+)}?", regex.UNICODE
     ),
+    # e.g. USE ${flyway:database}.schema_name;
+    "flyway_var": regex.compile(r"\${(?P<param_name>\w+[:\w_]+)}", regex.UNICODE),
     # e.g. WHERE bla = ?
     "question_mark": regex.compile(r"(?<![:\w\x5c])\?", regex.UNICODE),
     # e.g. WHERE bla = $3 or WHERE bla = ${3}
diff --git a/test/core/config_test.py b/test/core/config_test.py
index c67f86f30ae..59a84ce45a9 100644
--- a/test/core/config_test.py
+++ b/test/core/config_test.py
@@ -137,6 +137,28 @@ def test__config__load_toml():
     }
 
 
+def test__config__load_placeholder_cfg():
+    """Test loading a sqlfluff configuration file for placeholder templater."""
+    c = ConfigLoader()
+    cfg = c.load_config_file(
+        os.path.join("test", "fixtures", "config", "placeholder"),
+        ".sqlfluff-placeholder",
+    )
+    assert cfg == {
+        "core": {
+            "testing_val": "foobar",
+            "testing_int": 4,
+        },
+        "bar": {"foo": "barbar"},
+        "templater": {
+            "placeholder": {
+                "param_style": "flyway_var",
+                "flyway:database": "test_db",
+            }
+        },
+    }
+
+
 def test__config__iter_config_paths_right_order():
     """Test that config paths are fetched ordered by priority."""
     c = ConfigLoader()
diff --git a/test/core/templaters/placeholder_test.py b/test/core/templaters/placeholder_test.py
index d2541aacbbb..337104d679b 100644
--- a/test/core/templaters/placeholder_test.py
+++ b/test/core/templaters/placeholder_test.py
@@ -289,6 +289,22 @@ def test__templater_raw():
                 start_date="'2021-10-01'",
             ),
         ),
+        (
+            "USE ${flyway:database}.test_schema;",
+            "flyway_var",
+            "USE test_db.test_schema;",
+            {
+                "flyway:database": "test_db",
+            },
+        ),
+        (
+            "SELECT metadata$filename, $1 FROM @stg_data_export_${env_name};",
+            "flyway_var",
+            "SELECT metadata$filename, $1 FROM @stg_data_export_staging;",
+            {
+                "env_name": "staging",
+            },
+        ),
     ],
     ids=[
         "no_changes",
@@ -306,6 +322,8 @@ def test__templater_raw():
         "numeric_dollar_with_braces_and_string",
         "percent",
         "ampersand",
+        "flyway_var",
+        "flyway_var",
     ],
 )
 def test__templater_param_style(instr, expected_outstr, param_style, values):
diff --git a/test/fixtures/config/placeholder/.sqlfluff-placeholder b/test/fixtures/config/placeholder/.sqlfluff-placeholder
new file mode 100644
index 00000000000..fbc846084b2
--- /dev/null
+++ b/test/fixtures/config/placeholder/.sqlfluff-placeholder
@@ -0,0 +1,10 @@
+[sqlfluff]
+testing_val=foobar
+testing_int=4
+
+[sqlfluff:bar]
+foo=barbar
+
+[sqlfluff:templater:placeholder]
+param_style = flyway_var
+flyway:database = test_db
\ No newline at end of file
diff --git a/test/fixtures/templater/placeholder_flyway_var/placeholder_flyway_var_a.sql b/test/fixtures/templater/placeholder_flyway_var/placeholder_flyway_var_a.sql
new file mode 100644
index 00000000000..4d10e9d5e92
--- /dev/null
+++ b/test/fixtures/templater/placeholder_flyway_var/placeholder_flyway_var_a.sql
@@ -0,0 +1,5 @@
+USE ${flyway:database}.test_schema;
+
+CREATE OR REPLACE STAGE stg_data_export_${env_name}
+URL = 's3://${s3_data_lake_bucket}/${env_name}/exports/stg_data_export'
+STORAGE_INTEGRATION = s3_integ_main;