From 8793b425af2b3b0add0ee75d058e127e3b487edf Mon Sep 17 00:00:00 2001 From: Przemek Denkiewicz Date: Fri, 12 Aug 2022 11:57:25 +0200 Subject: [PATCH] Add persist_docs functionality --- dbt/include/trino/macros/adapters.sql | 14 ++ .../macros/materializations/incremental.sql | 4 +- .../trino/macros/materializations/table.sql | 2 + .../trino/macros/materializations/view.sql | 7 +- .../adapter/persist_docs/fixtures.py | 109 +++++++++++++ .../adapter/persist_docs/test_persist_docs.py | 145 ++++++++++++++++++ 6 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 tests/functional/adapter/persist_docs/fixtures.py create mode 100644 tests/functional/adapter/persist_docs/test_persist_docs.py diff --git a/dbt/include/trino/macros/adapters.sql b/dbt/include/trino/macros/adapters.sql index 747c7573..f0facf40 100644 --- a/dbt/include/trino/macros/adapters.sql +++ b/dbt/include/trino/macros/adapters.sql @@ -158,6 +158,20 @@ {% endmacro %} +{% macro trino__alter_relation_comment(relation, relation_comment) -%} + comment on {{ relation.type }} {{ relation }} is '{{ relation_comment | replace("'", "''") }}'; +{% endmacro %} + + +{% macro trino__alter_column_comment(relation, column_dict) %} + {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %} + {% for column_name in column_dict if (column_name in existing_columns) %} + {% set comment = column_dict[column_name]['description'] %} + comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is '{{ comment | replace("'", "''") }}'; + {% endfor %} +{% endmacro %} + + {% macro trino__list_schemas(database) -%} {% call statement('list_schemas', fetch_result=True, auto_begin=False) %} select distinct schema_name diff --git a/dbt/include/trino/macros/materializations/incremental.sql b/dbt/include/trino/macros/materializations/incremental.sql index a1a96069..be96096d 100644 --- a/dbt/include/trino/macros/materializations/incremental.sql +++ b/dbt/include/trino/macros/materializations/incremental.sql @@ -13,7 +13,7 @@ {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - {% set target_relation = this %} + {%- set target_relation = this.incorporate(type='table') -%} {% set existing_relation = load_relation(this) %} {% set tmp_relation = make_temp_relation(this) %} @@ -37,6 +37,8 @@ {{ run_hooks(post_hooks) }} + {% do persist_docs(target_relation, model) %} + {{ return({'relations': [target_relation]}) }} {%- endmaterialization %} diff --git a/dbt/include/trino/macros/materializations/table.sql b/dbt/include/trino/macros/materializations/table.sql index 154f2afa..86f44e00 100644 --- a/dbt/include/trino/macros/materializations/table.sql +++ b/dbt/include/trino/macros/materializations/table.sql @@ -67,5 +67,7 @@ {{ run_hooks(post_hooks) }} + {% do persist_docs(target_relation, model) %} + {{ return({'relations': [target_relation]}) }} {% endmaterialization %} diff --git a/dbt/include/trino/macros/materializations/view.sql b/dbt/include/trino/macros/materializations/view.sql index fe155b70..dbdbd537 100644 --- a/dbt/include/trino/macros/materializations/view.sql +++ b/dbt/include/trino/macros/materializations/view.sql @@ -1,3 +1,8 @@ {% materialization view, adapter='trino' -%} - {{ return(create_or_replace_view()) }} + {% set to_return = create_or_replace_view() %} + {% set target_relation = this.incorporate(type='view') %} + + {% do persist_docs(target_relation, model) %} + + {% do return(to_return) %} {%- endmaterialization %} diff --git a/tests/functional/adapter/persist_docs/fixtures.py b/tests/functional/adapter/persist_docs/fixtures.py new file mode 100644 index 00000000..25385f53 --- /dev/null +++ b/tests/functional/adapter/persist_docs/fixtures.py @@ -0,0 +1,109 @@ +seed_csv = """ +id,name,date +1,Easton,1981-05-20 06:46:51 +2,Lillian,1978-09-03 18:10:33 +3,Jeremiah,1982-03-11 03:59:51 +4,Nolan,1976-05-06 20:21:35 +""".lstrip() + +table_model = """ +{{config(materialized = "table")}} +select * from {{ ref('seed') }} +""" + +view_model = """ +{{config(materialized = "view")}} +select * from {{ ref('seed') }} +""" + +incremental_model = """ +{{config(materialized = "incremental")}} +select * from {{ ref('seed') }} +""" + +table_profile_yml = """ +version: 2 +models: + - name: table_model + description: | + Table model description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + columns: + - name: id + description: | + id Column description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + tests: + - unique + - not_null + - name: name + description: | + Fancy column description + tests: + - not_null +""" + + +view_profile_yml = """ +version: 2 +models: + - name: view_model + description: | + Table model description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + columns: + - name: id + tests: + - unique + - not_null + - name: name + tests: + - not_null +""" + +incremental_profile_yml = """ +version: 2 +models: + - name: incremental_model + description: | + Table model description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + columns: + - name: id + description: | + id Column description "with double quotes" + and with 'single quotes' as welll as other; + '''abc123''' + reserved -- characters + -- + /* comment */ + Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + tests: + - unique + - not_null + - name: name + description: | + Fancy column description + tests: + - not_null +""" diff --git a/tests/functional/adapter/persist_docs/test_persist_docs.py b/tests/functional/adapter/persist_docs/test_persist_docs.py new file mode 100644 index 00000000..8d3307cb --- /dev/null +++ b/tests/functional/adapter/persist_docs/test_persist_docs.py @@ -0,0 +1,145 @@ +import pytest +from dbt.tests.util import run_dbt, check_relations_equal, run_sql_with_adapter + +from tests.functional.adapter.persist_docs.fixtures import ( + seed_csv, + table_model, + view_model, + incremental_model, + table_profile_yml, + view_profile_yml, + incremental_profile_yml +) + + +@pytest.mark.iceberg +class TestPersistDocsBase: + """ + Testing persist_docs functionality + """ + + @property + def schema(self): + return "default" + + # everything that goes in the "seeds" directory + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": seed_csv, + } + + +class TestPersistDocsTable(TestPersistDocsBase): + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "persist_docs_tests", + "models": { + "+persist_docs": { + "relation": True, + "columns": True + } + }, + "seeds": { + "+column_types": { + "date": "timestamp(6)" + } + } + } + + # everything that goes in the "models" directory + @pytest.fixture(scope="class") + def models(self): + return { + "table_model.sql": table_model, + "table_persist_docs.yml": table_profile_yml, + } + + def test_run_seed_test(self, project): + # seed seeds + results = run_dbt(["seed"], expect_pass=True) + assert len(results) == 1 + results = run_dbt(["run"], expect_pass=True) + assert len(results) == 1 + # test tests + results = run_dbt(["test"], expect_pass=True) + assert len(results) == 3 + + +# Comments on views are not supported in Trino engine +# https://github.com/trinodb/trino/issues/10705 +class TestPersistDocsView(TestPersistDocsBase): + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "persist_docs_tests", + "models": { + "+persist_docs": { + "relation": True + } + }, + "seeds": { + "+column_types": { + "date": "timestamp(6)" + } + } + } + + # everything that goes in the "models" directory + @pytest.fixture(scope="class") + def models(self): + return { + "view_model.sql": view_model, + "view_persist_docs.yml": view_profile_yml, + } + + def test_run_seed_test(self, project): + # seed seeds + results = run_dbt(["seed"], expect_pass=True) + assert len(results) == 1 + results = run_dbt(["run"], expect_pass=True) + assert len(results) == 1 + # test tests + results = run_dbt(["test"], expect_pass=True) + assert len(results) == 3 + + +class TestPersistDocsIncremental(TestPersistDocsBase): + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "persist_docs_tests", + "models": { + "+persist_docs": { + "relation": True, + "columns": True + } + }, + "seeds": { + "+column_types": { + "date": "timestamp(6)" + } + } + } + + # everything that goes in the "models" directory + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_model.sql": incremental_model, + "incremental_persist_docs.yml": incremental_profile_yml, + } + + def test_run_seed_test(self, project): + # seed seeds + results = run_dbt(["seed"], expect_pass=True) + assert len(results) == 1 + results = run_dbt(["run"], expect_pass=True) + assert len(results) == 1 + # test tests + results = run_dbt(["test"], expect_pass=True) + assert len(results) == 3