From 1349974af3dcff7fbd830c586e7aca0294e4d59c Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 6 Feb 2023 15:22:16 -0700 Subject: [PATCH 1/3] add draft changes for addl conn_types --- data_diff/dbt.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/data_diff/dbt.py b/data_diff/dbt.py index c34d21d4..499d52f6 100644 --- a/data_diff/dbt.py +++ b/data_diff/dbt.py @@ -268,6 +268,7 @@ def set_connection(self): # values can contain env_vars rendered_credentials = ProfileRenderer().render_data(credentials) + # this whole block should be refactored/extracted to method(s) if conn_type == "snowflake": if rendered_credentials.get("password") is None or rendered_credentials.get("private_key_path") is not None: raise Exception("Only password authentication is currently supported for Snowflake.") @@ -293,6 +294,37 @@ def set_connection(self): "project": rendered_credentials.get("project"), "dataset": rendered_credentials.get("dataset"), } + # untested + elif conn_type == "redshift": + conn_info = { + "driver": conn_type, + "host": rendered_credentials.get("host"), + "user": rendered_credentials.get("user"), + "password": rendered_credentials.get("password"), + "port": rendered_credentials.get("port"), + "dbname": rendered_credentials.get("dbname") + } + # untested + elif conn_type == "databricks": + conn_info = { + "driver": conn_type, + "catalog": rendered_credentials.get("catalog"), + "server_hostname": rendered_credentials.get("host"), + # TODO may need to trim the / at http_path[0] + "http_path": rendered_credentials.get("http_path"), + "schema": rendered_credentials.get("schema"), + "access_token": rendered_credentials.get("token") + } + # untested + elif conn_type == "postgres": + conn_info ={ + "driver": "postgresql", + "host": rendered_credentials.get("host"), + "user": rendered_credentials.get("user"), + "password": rendered_credentials.get("password"), + "port": rendered_credentials.get("port"), + "dbname": rendered_credentials.get("dbname") or rendered_credentials.get("database"), + } else: raise NotImplementedError(f"Provider {conn_type} is not yet supported for dbt diffs") From ab5739435a12a447185b528bb86b8908d66e9a75 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 6 Feb 2023 15:25:43 -0700 Subject: [PATCH 2/3] black --- data_diff/dbt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data_diff/dbt.py b/data_diff/dbt.py index 499d52f6..334d48cb 100644 --- a/data_diff/dbt.py +++ b/data_diff/dbt.py @@ -302,7 +302,7 @@ def set_connection(self): "user": rendered_credentials.get("user"), "password": rendered_credentials.get("password"), "port": rendered_credentials.get("port"), - "dbname": rendered_credentials.get("dbname") + "dbname": rendered_credentials.get("dbname"), } # untested elif conn_type == "databricks": @@ -313,11 +313,11 @@ def set_connection(self): # TODO may need to trim the / at http_path[0] "http_path": rendered_credentials.get("http_path"), "schema": rendered_credentials.get("schema"), - "access_token": rendered_credentials.get("token") + "access_token": rendered_credentials.get("token"), } # untested elif conn_type == "postgres": - conn_info ={ + conn_info = { "driver": "postgresql", "host": rendered_credentials.get("host"), "user": rendered_credentials.get("user"), From dccdb1a657479f5349e291f525369f6caf82f82a Mon Sep 17 00:00:00 2001 From: Dan Date: Fri, 3 Mar 2023 15:26:58 -0700 Subject: [PATCH 3/3] test + cleanup --- data_diff/dbt.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/data_diff/dbt.py b/data_diff/dbt.py index 1fe2a159..d7042ed5 100644 --- a/data_diff/dbt.py +++ b/data_diff/dbt.py @@ -36,7 +36,7 @@ def import_dbt(): PROJECT_FILE = "/dbt_project.yml" PROFILES_FILE = "/profiles.yml" LOWER_DBT_V = "1.0.0" -UPPER_DBT_V = "1.5.0" +UPPER_DBT_V = "1.4.2" @dataclass @@ -343,7 +343,6 @@ def _get_connection_creds(self) -> Tuple[Dict[str, str], str]: def set_connection(self): rendered_credentials, conn_type = self._get_connection_creds() - # this whole block should be refactored/extracted to method(s) if conn_type == "snowflake": if rendered_credentials.get("password") is None or rendered_credentials.get("private_key_path") is not None: raise Exception("Only password authentication is currently supported for Snowflake.") @@ -369,8 +368,9 @@ def set_connection(self): "project": rendered_credentials.get("project"), "dataset": rendered_credentials.get("dataset"), } - # untested elif conn_type == "redshift": + if rendered_credentials.get("password") is None or rendered_credentials.get("method") == "iam": + raise Exception("Only password authentication is currently supported for Redshift.") conn_info = { "driver": conn_type, "host": rendered_credentials.get("host"), @@ -379,18 +379,15 @@ def set_connection(self): "port": rendered_credentials.get("port"), "dbname": rendered_credentials.get("dbname"), } - # untested elif conn_type == "databricks": conn_info = { "driver": conn_type, "catalog": rendered_credentials.get("catalog"), "server_hostname": rendered_credentials.get("host"), - # TODO may need to trim the / at http_path[0] "http_path": rendered_credentials.get("http_path"), "schema": rendered_credentials.get("schema"), "access_token": rendered_credentials.get("token"), } - # untested elif conn_type == "postgres": conn_info = { "driver": "postgresql",