Skip to content

Commit

Permalink
fix_insert_overwrite (#164)
Browse files Browse the repository at this point in the history
  • Loading branch information
vamshikolanu committed May 2, 2023
1 parent 180bb9c commit a126fc3
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 4 deletions.
3 changes: 0 additions & 3 deletions dbt/adapters/impala/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,9 +191,6 @@ def parse_describe_extended(
table_separator_pos = ImpalaAdapter.find_table_information_separator(dict_rows) # ensure that the class method is called

column_separator_pos = partition_separator_pos if partition_separator_pos > 0 else table_separator_pos
print('table_separator_pos', table_separator_pos)
print('partition_separator_pos', partition_separator_pos)
print('column_separator_pos', column_separator_pos)
# Remove rows that start with a hash, they are comments
rows = [
row for row in raw_rows[0:column_separator_pos]
Expand Down
2 changes: 1 addition & 1 deletion dbt/include/impala/macros/insertoverwrite.sql
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

{% macro get_insert_overwrite_sql(target, source, dest_columns) -%}

{%- set raw_strategy = config.get('incremental_strategy', default='append') -%}
{% set raw_strategy = config.get('incremental_strategy') or 'append' %}
{%- set partition_cols = config.get('partition_by', validator=validation.any[list]) -%}

{% if partition_cols is not none and raw_strategy == 'insert_overwrite' %}
Expand Down
13 changes: 13 additions & 0 deletions tests/functional/adapter/test_basic.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,19 @@ def test_incremental(self, project):
assert len(catalog.nodes) == 3
assert len(catalog.sources) == 1

insertoverwrite_sql = """
{{ config(materialized="incremental", incremental_strategy="insert_overwrite", partition_by="id_partition") }}
select *, id as id_partition from {{ source('raw', 'seed') }}
{% if is_incremental() %}
where id > (select max(id) from {{ this }})
{% endif %}
""".strip()

class TestInsertoverwriteImpala(TestIncrementalImpala):
@pytest.fixture(scope="class")
def models(self):
return {"incremental_test_model.sql": insertoverwrite_sql, "schema.yml": schema_base_yml}

incremental_single_partitionby_sql = """
{{ config(materialized="incremental", partition_by="id_partition") }}
select *, id as id_partition from {{ source('raw', 'seed') }}
Expand Down

0 comments on commit a126fc3

Please sign in to comment.