From a41e90594f8f1651a655f159ff4a042ac88b9404 Mon Sep 17 00:00:00 2001 From: Damian Owsianny Date: Fri, 14 Apr 2023 11:35:22 +0200 Subject: [PATCH 1/4] Bumping version to 1.5.0rc1 --- dbt/adapters/trino/__version__.py | 2 +- dev_requirements.txt | 2 +- tests/unit/utils.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/trino/__version__.py b/dbt/adapters/trino/__version__.py index 841aad2c..fa6c5a1a 100644 --- a/dbt/adapters/trino/__version__.py +++ b/dbt/adapters/trino/__version__.py @@ -1 +1 @@ -version = "1.4.2" +version = "1.5.0rc1" diff --git a/dev_requirements.txt b/dev_requirements.txt index 8ad57e1d..039eb398 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,4 +1,4 @@ -dbt-tests-adapter~=1.4.5 +dbt-tests-adapter~=1.5.0rc1 mypy==1.2.0 # patch updates have historically introduced breaking changes pre-commit~=3.2 pytest~=7.2 diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 2f719066..1778de1b 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -43,6 +43,14 @@ def profile_from_dict(profile, profile_name, cli_vars="{}"): cli_vars = parse_cli_vars(cli_vars) renderer = ProfileRenderer(cli_vars) + + # in order to call dbt's internal profile rendering, we need to set the + # flags global. This is a bit of a hack, but it's the best way to do it. + from argparse import Namespace + + from dbt.flags import set_from_args + + set_from_args(Namespace(), None) return Profile.from_raw_profile_info( profile, profile_name, @@ -74,6 +82,10 @@ def config_from_parts_or_dicts(project, profile, packages=None, selectors=None, from copy import deepcopy from dbt.config import Profile, Project, RuntimeConfig + from dbt.config.utils import parse_cli_vars + + if not isinstance(cli_vars, dict): + cli_vars = parse_cli_vars(cli_vars) if isinstance(project, Project): profile_name = project.profile_name From dc2afc90c065888dafcd2ed3dc918b50fc7a7ad5 Mon Sep 17 00:00:00 2001 From: Damian Owsianny Date: Tue, 18 Apr 2023 16:33:29 +0200 Subject: [PATCH 2/4] Add model contracts --- .../unreleased/Features-20230419-115208.yaml | 7 + dbt/adapters/trino/connections.py | 4 + dbt/adapters/trino/impl.py | 11 +- dbt/include/trino/macros/adapters.sql | 34 ++- .../adapter/constraints/fixtures.py | 119 ++++++++++ .../adapter/constraints/test_constraints.py | 213 ++++++++++++++++++ 6 files changed, 383 insertions(+), 5 deletions(-) create mode 100644 .changes/unreleased/Features-20230419-115208.yaml create mode 100644 tests/functional/adapter/constraints/fixtures.py create mode 100644 tests/functional/adapter/constraints/test_constraints.py diff --git a/.changes/unreleased/Features-20230419-115208.yaml b/.changes/unreleased/Features-20230419-115208.yaml new file mode 100644 index 00000000..4aa85a9f --- /dev/null +++ b/.changes/unreleased/Features-20230419-115208.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Add model contracts +time: 2023-04-19T11:52:08.343309+02:00 +custom: + Author: damian3031 + Issue: "284" + PR: "286" diff --git a/dbt/adapters/trino/connections.py b/dbt/adapters/trino/connections.py index 9313e2ed..774140af 100644 --- a/dbt/adapters/trino/connections.py +++ b/dbt/adapters/trino/connections.py @@ -508,3 +508,7 @@ def execute(self, sql, auto_begin=False, fetch=False): status = self.get_response(cursor) table = self.get_result_from_cursor(cursor) return status, table + + @classmethod + def data_type_code_to_name(cls, type_code) -> str: + return type_code.split("(")[0].upper() diff --git a/dbt/adapters/trino/impl.py b/dbt/adapters/trino/impl.py index ec6c5ba8..b276b616 100644 --- a/dbt/adapters/trino/impl.py +++ b/dbt/adapters/trino/impl.py @@ -2,8 +2,9 @@ from typing import Dict, Optional import agate -from dbt.adapters.base.impl import AdapterConfig +from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport from dbt.adapters.sql import SQLAdapter +from dbt.contracts.graph.nodes import ConstraintType from dbt.exceptions import DbtDatabaseError from dbt.adapters.trino import TrinoColumn, TrinoConnectionManager, TrinoRelation @@ -21,6 +22,14 @@ class TrinoAdapter(SQLAdapter): ConnectionManager = TrinoConnectionManager AdapterSpecificConfigs = TrinoConfig + CONSTRAINT_SUPPORT = { + ConstraintType.check: ConstraintSupport.NOT_SUPPORTED, + ConstraintType.not_null: ConstraintSupport.NOT_SUPPORTED, + ConstraintType.unique: ConstraintSupport.NOT_SUPPORTED, + ConstraintType.primary_key: ConstraintSupport.NOT_SUPPORTED, + ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, + } + @classmethod def date_function(cls): return "datenow()" diff --git a/dbt/include/trino/macros/adapters.sql b/dbt/include/trino/macros/adapters.sql index 542e519e..fb6d0020 100644 --- a/dbt/include/trino/macros/adapters.sql +++ b/dbt/include/trino/macros/adapters.sql @@ -91,11 +91,33 @@ {% macro trino__create_table_as(temporary, relation, sql) -%} {%- set _properties = config.get('properties') -%} - create table {{ relation }} + + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + + create table + {{ relation }} + {{ get_table_columns_and_constraints() }} + {{ get_assert_columns_equivalent(sql) }} + {%- set sql = get_select_subquery(sql) %} {{ properties(_properties) }} - as ( - {{ sql }} - ); + ; + + insert into {{ relation }} + ( + {{ sql }} + ) + ; + + {%- else %} + + create table {{ relation }} + {{ properties(_properties) }} + as ( + {{ sql }} + ); + + {%- endif %} {% endmacro %} @@ -108,6 +130,10 @@ {% endif %} create or replace view {{ relation }} + {%- set contract_config = config.get('contract') -%} + {%- if contract_config.enforced -%} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} security {{ view_security }} as {{ sql }} diff --git a/tests/functional/adapter/constraints/fixtures.py b/tests/functional/adapter/constraints/fixtures.py new file mode 100644 index 00000000..010f0285 --- /dev/null +++ b/tests/functional/adapter/constraints/fixtures.py @@ -0,0 +1,119 @@ +# model breaking constraints +trino_model_char_value_to_int_column = """ +{{ + config( + materialized = "table" + ) +}} + +select + -- char value for 'id', which is integer type + 'char_value' as id, + -- change the color as well (to test rollback) + 'red' as color, + '2019-01-01' as date_day +""" + +trino_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + columns: + - name: id + quote: true + data_type: integer + description: hello + constraints: + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: varchar + - name: date_day + data_type: varchar + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: varchar + - name: date_day + data_type: varchar + - name: my_model_wrong_order + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: varchar + - name: date_day + data_type: varchar + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + description: hello + constraints: + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: varchar + - name: date_day + data_type: varchar +""" + +trino_constrained_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: check + expression: (id > 0) + - type: primary_key + columns: [ id ] + - type: unique + columns: [ color, date_day ] + name: strange_uniqueness_requirement + columns: + - name: id + quote: true + data_type: integer + description: hello + constraints: + - type: not_null + tests: + - unique + - name: color + data_type: varchar + - name: date_day + data_type: varchar +""" diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py new file mode 100644 index 00000000..4e253545 --- /dev/null +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -0,0 +1,213 @@ +import pytest +from dbt.tests.adapter.constraints.fixtures import ( + my_incremental_model_sql, + my_model_incremental_wrong_name_sql, + my_model_incremental_wrong_order_sql, + my_model_sql, + my_model_view_wrong_name_sql, + my_model_view_wrong_order_sql, + my_model_wrong_name_sql, + my_model_wrong_order_sql, +) +from dbt.tests.adapter.constraints.test_constraints import ( + BaseConstraintsRollback, + BaseConstraintsRuntimeDdlEnforcement, + BaseIncrementalConstraintsColumnsEqual, + BaseIncrementalConstraintsRollback, + BaseIncrementalConstraintsRuntimeDdlEnforcement, + BaseModelConstraintsRuntimeEnforcement, + BaseTableConstraintsColumnsEqual, + BaseViewConstraintsColumnsEqual, +) + +from tests.functional.adapter.constraints.fixtures import ( + trino_constrained_model_schema_yml, + trino_model_char_value_to_int_column, + trino_model_schema_yml, +) + +_expected_sql_trino = """ +create table ( + id integer, + color varchar, + date_day varchar +) ; +insert into +( + select + id, + color, + date_day from + ( + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + ) as model_subq +) +; +""" + + +class TrinoColumnEqualSetup: + @pytest.fixture + def string_type(self): + return "VARCHAR" + + @pytest.fixture + def data_types(self, schema_int_type, int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ["'1'", string_type, string_type], + ["cast('2019-01-01' as date)", "date", "DATE"], + ["true", "boolean", "BOOLEAN"], + ["cast('2013-11-03 00:00:00-07' as TIMESTAMP)", "timestamp(6)", "TIMESTAMP"], + [ + "cast('2013-11-03 00:00:00-07' as TIMESTAMP WITH TIME ZONE)", + "timestamp(6)", + "TIMESTAMP", + ], + ["ARRAY['a','b','c']", "ARRAY(VARCHAR)", "ARRAY"], + ["ARRAY[1,2,3]", "ARRAY(INTEGER)", "ARRAY"], + ["cast('1' as DECIMAL)", "DECIMAL", "DECIMAL"], + ] + + +class TestTrinoTableConstraintsColumnsEqual( + TrinoColumnEqualSetup, BaseTableConstraintsColumnsEqual +): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": trino_model_schema_yml, + } + + +class TestTrinoViewConstraintsColumnsEqual(TrinoColumnEqualSetup, BaseViewConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_view_wrong_order_sql, + "my_model_wrong_name.sql": my_model_view_wrong_name_sql, + "constraints_schema.yml": trino_model_schema_yml, + } + + +class TestTrinoIncrementalConstraintsColumnsEqual( + TrinoColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual +): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "constraints_schema.yml": trino_model_schema_yml, + } + + +class TestTrinoTableConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_wrong_order_sql, + "constraints_schema.yml": trino_model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return _expected_sql_trino + + +class TestTrinoTableConstraintsRollback(BaseConstraintsRollback): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": trino_model_schema_yml, + } + + # We are trying to load char value to integer column to break constraint. + # In BaseConstraintsRollback it is checked by violating not-null constraint, + # But not-null constraint is not supported in dbt-trino + @pytest.fixture(scope="class") + def null_model_sql(self): + return trino_model_char_value_to_int_column + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return [ + "Please ensure the name, data_type, and number of columns in your contract match the columns in your model's definition." + ] + + +class TestTrinoIncrementalConstraintsRuntimeDdlEnforcement( + BaseIncrementalConstraintsRuntimeDdlEnforcement +): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_incremental_wrong_order_sql, + "constraints_schema.yml": trino_model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return _expected_sql_trino + + +class TestTrinoIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": trino_model_schema_yml, + } + + # We are trying to load char value to integer column to break constraint. + # In BaseConstraintsRollback it is checked by violating not-null constraint, + # But not-null constraint is not supported in dbt-trino + @pytest.fixture(scope="class") + def null_model_sql(self): + return trino_model_char_value_to_int_column + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return [ + "Please ensure the name, data_type, and number of columns in your contract match the columns in your model's definition." + ] + + +class TestTrinoModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": trino_constrained_model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ +create table ( + id integer, + color varchar, + date_day varchar +) ; +insert into +( + select + id, + color, + date_day from + ( + select + 1 as id, + 'blue' as color, + '2019-01-01' as date_day + ) as model_subq +) +; +""" From 43a98f4403fc406d313c67a3aa273d1e0463c5e4 Mon Sep 17 00:00:00 2001 From: Damian Owsianny Date: Mon, 24 Apr 2023 11:15:52 +0200 Subject: [PATCH 3/4] Bumping version to 1.5.0rc2 --- dbt/adapters/trino/__version__.py | 2 +- dev_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/trino/__version__.py b/dbt/adapters/trino/__version__.py index fa6c5a1a..ab9ba8df 100644 --- a/dbt/adapters/trino/__version__.py +++ b/dbt/adapters/trino/__version__.py @@ -1 +1 @@ -version = "1.5.0rc1" +version = "1.5.0rc2" diff --git a/dev_requirements.txt b/dev_requirements.txt index 039eb398..f818a1a1 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,4 +1,4 @@ -dbt-tests-adapter~=1.5.0rc1 +dbt-tests-adapter~=1.5.0rc2 mypy==1.2.0 # patch updates have historically introduced breaking changes pre-commit~=3.2 pytest~=7.2 From 02b606f5ac95b1a5508a00e80fb4e6feca095ad5 Mon Sep 17 00:00:00 2001 From: Damian Owsianny Date: Wed, 26 Apr 2023 13:29:55 +0200 Subject: [PATCH 4/4] Support not-null constraint in model contracts --- dbt/adapters/trino/impl.py | 2 +- .../adapter/constraints/fixtures.py | 20 +++-------- .../adapter/constraints/test_constraints.py | 34 ++++++------------- 3 files changed, 16 insertions(+), 40 deletions(-) diff --git a/dbt/adapters/trino/impl.py b/dbt/adapters/trino/impl.py index b276b616..658f106b 100644 --- a/dbt/adapters/trino/impl.py +++ b/dbt/adapters/trino/impl.py @@ -24,7 +24,7 @@ class TrinoAdapter(SQLAdapter): CONSTRAINT_SUPPORT = { ConstraintType.check: ConstraintSupport.NOT_SUPPORTED, - ConstraintType.not_null: ConstraintSupport.NOT_SUPPORTED, + ConstraintType.not_null: ConstraintSupport.ENFORCED, ConstraintType.unique: ConstraintSupport.NOT_SUPPORTED, ConstraintType.primary_key: ConstraintSupport.NOT_SUPPORTED, ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, diff --git a/tests/functional/adapter/constraints/fixtures.py b/tests/functional/adapter/constraints/fixtures.py index 010f0285..26c4f60e 100644 --- a/tests/functional/adapter/constraints/fixtures.py +++ b/tests/functional/adapter/constraints/fixtures.py @@ -1,19 +1,3 @@ -# model breaking constraints -trino_model_char_value_to_int_column = """ -{{ - config( - materialized = "table" - ) -}} - -select - -- char value for 'id', which is integer type - 'char_value' as id, - -- change the color as well (to test rollback) - 'red' as color, - '2019-01-01' as date_day -""" - trino_model_schema_yml = """ version: 2 models: @@ -27,6 +11,7 @@ data_type: integer description: hello constraints: + - type: not_null - type: check expression: (id > 0) tests: @@ -44,6 +29,7 @@ data_type: integer description: hello constraints: + - type: not_null - type: check expression: (id > 0) tests: @@ -61,6 +47,7 @@ data_type: integer description: hello constraints: + - type: not_null - type: check expression: (id > 0) tests: @@ -78,6 +65,7 @@ data_type: integer description: hello constraints: + - type: not_null - type: check expression: (id > 0) tests: diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py index 4e253545..5ced292a 100644 --- a/tests/functional/adapter/constraints/test_constraints.py +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -22,13 +22,12 @@ from tests.functional.adapter.constraints.fixtures import ( trino_constrained_model_schema_yml, - trino_model_char_value_to_int_column, trino_model_schema_yml, ) _expected_sql_trino = """ create table ( - id integer, + id integer not null, color varchar, date_day varchar ) ; @@ -74,6 +73,7 @@ def data_types(self, schema_int_type, int_type, string_type): ] +@pytest.mark.iceberg class TestTrinoTableConstraintsColumnsEqual( TrinoColumnEqualSetup, BaseTableConstraintsColumnsEqual ): @@ -96,6 +96,7 @@ def models(self): } +@pytest.mark.iceberg class TestTrinoIncrementalConstraintsColumnsEqual( TrinoColumnEqualSetup, BaseIncrementalConstraintsColumnsEqual ): @@ -108,6 +109,7 @@ def models(self): } +@pytest.mark.iceberg class TestTrinoTableConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): @pytest.fixture(scope="class") def models(self): @@ -121,6 +123,7 @@ def expected_sql(self): return _expected_sql_trino +@pytest.mark.iceberg class TestTrinoTableConstraintsRollback(BaseConstraintsRollback): @pytest.fixture(scope="class") def models(self): @@ -129,20 +132,12 @@ def models(self): "constraints_schema.yml": trino_model_schema_yml, } - # We are trying to load char value to integer column to break constraint. - # In BaseConstraintsRollback it is checked by violating not-null constraint, - # But not-null constraint is not supported in dbt-trino - @pytest.fixture(scope="class") - def null_model_sql(self): - return trino_model_char_value_to_int_column - @pytest.fixture(scope="class") def expected_error_messages(self): - return [ - "Please ensure the name, data_type, and number of columns in your contract match the columns in your model's definition." - ] + return ["NULL value not allowed for NOT NULL column: id"] +@pytest.mark.iceberg class TestTrinoIncrementalConstraintsRuntimeDdlEnforcement( BaseIncrementalConstraintsRuntimeDdlEnforcement ): @@ -158,6 +153,7 @@ def expected_sql(self): return _expected_sql_trino +@pytest.mark.iceberg class TestTrinoIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): @pytest.fixture(scope="class") def models(self): @@ -166,20 +162,12 @@ def models(self): "constraints_schema.yml": trino_model_schema_yml, } - # We are trying to load char value to integer column to break constraint. - # In BaseConstraintsRollback it is checked by violating not-null constraint, - # But not-null constraint is not supported in dbt-trino - @pytest.fixture(scope="class") - def null_model_sql(self): - return trino_model_char_value_to_int_column - @pytest.fixture(scope="class") def expected_error_messages(self): - return [ - "Please ensure the name, data_type, and number of columns in your contract match the columns in your model's definition." - ] + return ["NULL value not allowed for NOT NULL column: id"] +@pytest.mark.iceberg class TestTrinoModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): @pytest.fixture(scope="class") def models(self): @@ -192,7 +180,7 @@ def models(self): def expected_sql(self): return """ create table ( - id integer, + id integer not null, color varchar, date_day varchar ) ;