override get strategi to make config options insensitive

Upsolver · May 19, 2023 · 4a63228 · 4a63228
1 parent b0f043b
commit 4a63228
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 18 deletions.
diff --git a/dbt/adapters/upsolver/impl.py b/dbt/adapters/upsolver/impl.py
@@ -49,7 +49,7 @@ def get_connection_from_sql(self, sql):
                                       .translate(str.maketrans({'\"':'', '\'':''}))
             return connection_identifier
         except Exception:
-            raise dbt.exceptions.ParsingError(f"Error while parsing connection name from sql:\n{sql}")
+            raise dbt.exceptions.ParsingError(f"Error while parsing connection name from sql: {sql}")
 
     @available
     def get_columns_names_with_types(self, list_dict):
@@ -95,6 +95,22 @@ def filter_options(self, options, parametr):
         editable = {key:val for key, val in options.items() if val[parametr] == True}
         return editable
 
+    @available
+    def get(self, config, key, default=None):
+        config = {k.lower(): v for k, v in config.items()}
+        value = config.get(key, default)
+        return value
+
+    @available
+    def require(self, config, key):
+        config = {k.lower(): v for k, v in config.items()}
+        value = config.get(key, None)
+        if value:
+            return value
+        else:
+            raise dbt.exceptions.ParsingError(f"Required option is missing: {key}")
+
+
     def get_options(self, source, options_type):
         if options_type == 'connection_options':
             options = Connection_options[source.lower()]

diff --git a/dbt/include/upsolver/macros/materializations/connection.sql b/dbt/include/upsolver/macros/materializations/connection.sql
@@ -1,8 +1,8 @@
 {% materialization connection, adapter='upsolver' %}
   {%- set identifier = model['alias'] -%}
-
-  {% set connection_type = config.require('connection_type') %}
-  {% set connection_options = config.require('connection_options') %}
+  {%- set model_config = model['config'] -%}
+  {% set connection_type = adapter.require(model_config, 'connection_type') %}
+  {% set connection_options = adapter.require(model_config, 'connection_options') %}
   {% set enriched_options = adapter.enrich_options(connection_options, connection_type, 'connection_options') %}
   {% set enriched_editable_options = adapter.filter_options(enriched_options, 'editable') %}
 

diff --git a/dbt/include/upsolver/macros/materializations/incremental/create_copy_job.sql b/dbt/include/upsolver/macros/materializations/incremental/create_copy_job.sql
@@ -4,6 +4,7 @@
     {% set job_options, source_options = adapter.separate_options(options, source) %}
     {%- if target_type != 'datalake' -%}
       {% set target_options = adapter.enrich_options(options, target_type, 'target_options') %}
+      {% set target_type = target_type %}
     {%- else -%}
       {% set target_options = {} %}
       {% set target_type = '' %}

diff --git a/dbt/include/upsolver/macros/materializations/incremental/incremental.sql b/dbt/include/upsolver/macros/materializations/incremental/incremental.sql
@@ -1,18 +1,18 @@
 {% materialization incremental, adapter='upsolver' %}
 
   {%- set identifier = model['alias'] -%}
-  {% set incremental_strategy = config.get('incremental_strategy', False) %}
-  {% set partition_by = config.get('partition_by', []) %}
-  {% set sync = config.get('sync', False) %}
-  {% set options = config.get('options', {}) %}
-  {% set source = config.get('source', none) %}
-  {% set target_type = config.get('target_type', 'datalake') %}
-  {% set target_connection = config.get('target_connection', none) %}
-  {% set target_schema = config.get('target_schema', schema) %}
-  {% set delete_condition = config.get('delete_condition', False) %}
-  {% set partition_by = config.get('partition_by', []) %}
-  {% set primary_key = config.get('primary_key', []) %}
-  {% set map_columns_by_name = config.get('map_columns_by_name', False) %}
+  {%- set model_config = model['config'] -%}
+  {% set incremental_strategy = adapter.get(model_config, 'incremental_strategy', False) %}
+  {% set sync = adapter.get(model_config, 'sync', False) %}
+  {% set options = adapter.get(model_config, 'options', {}) %}
+  {% set source = adapter.get(model_config, 'source') %}
+  {% set target_type = adapter.require(model_config, 'target_type').lower() %}
+  {% set target_connection = adapter.get(model_config, 'target_connection') %}
+  {% set target_schema = adapter.get(model_config, 'target_schema', schema) %}
+  {% set delete_condition = adapter.get(model_config, 'delete_condition', False) %}
+  {% set partition_by = adapter.get(model_config, 'partition_by', []) %}
+  {% set primary_key = adapter.get(model_config, 'primary_key', []) %}
+  {% set map_columns_by_name = adapter.get(model_config, 'map_columns_by_name', False) %}
   {% set job_identifier = identifier + '_job' %}
 
   {%- set old_relation = adapter.get_relation(identifier=job_identifier,
@@ -26,6 +26,7 @@
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
   {{ run_hooks(pre_hooks, inside_transaction=True) }}
+  {{ log("model[config]: " ~ model['config'] ) }}
 
 
   {% if target_type  == 'datalake' %}

diff --git a/dbt/include/upsolver/macros/materializations/materializedview.sql b/dbt/include/upsolver/macros/materializations/materializedview.sql
@@ -1,7 +1,8 @@
 {% materialization materializedview, adapter='upsolver' %}
   {%- set identifier = model['alias'] -%}
-  {% set sync = config.get('sync', False) %}
-  {% set options = config.get('options', {}) %}
+  {%- set model_config = model['config'] -%}
+  {% set sync = adapter.get(model_config, 'sync', False) %}
+  {% set options = adapter.get(model_config, 'options', {}) %}
   {% set enriched_options = adapter.enrich_options(options, materialized_view, 'target_options') %}
   {% set enriched_editable_options = adapter.filter_options(enriched_options, 'editable') %}