Skip to content

Commit

Permalink
Add persist_docs functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
hovaesco committed Aug 12, 2022
1 parent 054b067 commit 8793b42
Show file tree
Hide file tree
Showing 6 changed files with 279 additions and 2 deletions.
14 changes: 14 additions & 0 deletions dbt/include/trino/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,20 @@
{% endmacro %}


{% macro trino__alter_relation_comment(relation, relation_comment) -%}
comment on {{ relation.type }} {{ relation }} is '{{ relation_comment | replace("'", "''") }}';
{% endmacro %}
{% macro trino__alter_column_comment(relation, column_dict) %}
{% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %}
{% for column_name in column_dict if (column_name in existing_columns) %}
{% set comment = column_dict[column_name]['description'] %}
comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is '{{ comment | replace("'", "''") }}';
{% endfor %}
{% endmacro %}


{% macro trino__list_schemas(database) -%}
{% call statement('list_schemas', fetch_result=True, auto_begin=False) %}
select distinct schema_name
Expand Down
4 changes: 3 additions & 1 deletion dbt/include/trino/macros/materializations/incremental.sql
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

{%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%}

{% set target_relation = this %}
{%- set target_relation = this.incorporate(type='table') -%}
{% set existing_relation = load_relation(this) %}
{% set tmp_relation = make_temp_relation(this) %}

Expand All @@ -37,6 +37,8 @@

{{ run_hooks(post_hooks) }}

{% do persist_docs(target_relation, model) %}

{{ return({'relations': [target_relation]}) }}

{%- endmaterialization %}
2 changes: 2 additions & 0 deletions dbt/include/trino/macros/materializations/table.sql
Original file line number Diff line number Diff line change
Expand Up @@ -67,5 +67,7 @@

{{ run_hooks(post_hooks) }}

{% do persist_docs(target_relation, model) %}

{{ return({'relations': [target_relation]}) }}
{% endmaterialization %}
7 changes: 6 additions & 1 deletion dbt/include/trino/macros/materializations/view.sql
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
{% materialization view, adapter='trino' -%}
{{ return(create_or_replace_view()) }}
{% set to_return = create_or_replace_view() %}
{% set target_relation = this.incorporate(type='view') %}

{% do persist_docs(target_relation, model) %}

{% do return(to_return) %}
{%- endmaterialization %}
109 changes: 109 additions & 0 deletions tests/functional/adapter/persist_docs/fixtures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
seed_csv = """
id,name,date
1,Easton,1981-05-20 06:46:51
2,Lillian,1978-09-03 18:10:33
3,Jeremiah,1982-03-11 03:59:51
4,Nolan,1976-05-06 20:21:35
""".lstrip()

table_model = """
{{config(materialized = "table")}}
select * from {{ ref('seed') }}
"""

view_model = """
{{config(materialized = "view")}}
select * from {{ ref('seed') }}
"""

incremental_model = """
{{config(materialized = "incremental")}}
select * from {{ ref('seed') }}
"""

table_profile_yml = """
version: 2
models:
- name: table_model
description: |
Table model description "with double quotes"
and with 'single quotes' as welll as other;
'''abc123'''
reserved -- characters
--
/* comment */
Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
columns:
- name: id
description: |
id Column description "with double quotes"
and with 'single quotes' as welll as other;
'''abc123'''
reserved -- characters
--
/* comment */
Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
tests:
- unique
- not_null
- name: name
description: |
Fancy column description
tests:
- not_null
"""


view_profile_yml = """
version: 2
models:
- name: view_model
description: |
Table model description "with double quotes"
and with 'single quotes' as welll as other;
'''abc123'''
reserved -- characters
--
/* comment */
Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
columns:
- name: id
tests:
- unique
- not_null
- name: name
tests:
- not_null
"""

incremental_profile_yml = """
version: 2
models:
- name: incremental_model
description: |
Table model description "with double quotes"
and with 'single quotes' as welll as other;
'''abc123'''
reserved -- characters
--
/* comment */
Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
columns:
- name: id
description: |
id Column description "with double quotes"
and with 'single quotes' as welll as other;
'''abc123'''
reserved -- characters
--
/* comment */
Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
tests:
- unique
- not_null
- name: name
description: |
Fancy column description
tests:
- not_null
"""
145 changes: 145 additions & 0 deletions tests/functional/adapter/persist_docs/test_persist_docs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
import pytest
from dbt.tests.util import run_dbt, check_relations_equal, run_sql_with_adapter

from tests.functional.adapter.persist_docs.fixtures import (
seed_csv,
table_model,
view_model,
incremental_model,
table_profile_yml,
view_profile_yml,
incremental_profile_yml
)


@pytest.mark.iceberg
class TestPersistDocsBase:
"""
Testing persist_docs functionality
"""

@property
def schema(self):
return "default"

# everything that goes in the "seeds" directory
@pytest.fixture(scope="class")
def seeds(self):
return {
"seed.csv": seed_csv,
}


class TestPersistDocsTable(TestPersistDocsBase):

@pytest.fixture(scope="class")
def project_config_update(self):
return {
"name": "persist_docs_tests",
"models": {
"+persist_docs": {
"relation": True,
"columns": True
}
},
"seeds": {
"+column_types": {
"date": "timestamp(6)"
}
}
}

# everything that goes in the "models" directory
@pytest.fixture(scope="class")
def models(self):
return {
"table_model.sql": table_model,
"table_persist_docs.yml": table_profile_yml,
}

def test_run_seed_test(self, project):
# seed seeds
results = run_dbt(["seed"], expect_pass=True)
assert len(results) == 1
results = run_dbt(["run"], expect_pass=True)
assert len(results) == 1
# test tests
results = run_dbt(["test"], expect_pass=True)
assert len(results) == 3


# Comments on views are not supported in Trino engine
# https://github.com/trinodb/trino/issues/10705
class TestPersistDocsView(TestPersistDocsBase):

@pytest.fixture(scope="class")
def project_config_update(self):
return {
"name": "persist_docs_tests",
"models": {
"+persist_docs": {
"relation": True
}
},
"seeds": {
"+column_types": {
"date": "timestamp(6)"
}
}
}

# everything that goes in the "models" directory
@pytest.fixture(scope="class")
def models(self):
return {
"view_model.sql": view_model,
"view_persist_docs.yml": view_profile_yml,
}

def test_run_seed_test(self, project):
# seed seeds
results = run_dbt(["seed"], expect_pass=True)
assert len(results) == 1
results = run_dbt(["run"], expect_pass=True)
assert len(results) == 1
# test tests
results = run_dbt(["test"], expect_pass=True)
assert len(results) == 3


class TestPersistDocsIncremental(TestPersistDocsBase):

@pytest.fixture(scope="class")
def project_config_update(self):
return {
"name": "persist_docs_tests",
"models": {
"+persist_docs": {
"relation": True,
"columns": True
}
},
"seeds": {
"+column_types": {
"date": "timestamp(6)"
}
}
}

# everything that goes in the "models" directory
@pytest.fixture(scope="class")
def models(self):
return {
"incremental_model.sql": incremental_model,
"incremental_persist_docs.yml": incremental_profile_yml,
}

def test_run_seed_test(self, project):
# seed seeds
results = run_dbt(["seed"], expect_pass=True)
assert len(results) == 1
results = run_dbt(["run"], expect_pass=True)
assert len(results) == 1
# test tests
results = run_dbt(["test"], expect_pass=True)
assert len(results) == 3

0 comments on commit 8793b42

Please sign in to comment.