From 305874c19abbee6ab0e765322267f8e4f3e42071 Mon Sep 17 00:00:00 2001 From: Raymond Cypher Date: Mon, 1 May 2023 14:19:02 -0600 Subject: [PATCH] Throw error if model contract enforced (#322) * Mirror of dbt-databricks#319 but rebased on 1.5.test branch Signed-off-by: Jesse Whitehouse * Throw error if model contract enforced Until we support dbt 1.5 model contracts we will throw an error if contract->enforced is set to true for a model. Added doc outlining differences between dbt-databricks constraints and dbt model contract. Signed-off-by: Raymond Cypher * Update change log Signed-off-by: Raymond Cypher * added missing newline Signed-off-by: Raymond Cypher * Add all columns to model definition Signed-off-by: Raymond Cypher --------- Signed-off-by: Jesse Whitehouse Signed-off-by: Raymond Cypher Co-authored-by: Jesse Whitehouse --- CHANGELOG.md | 3 + dbt/include/databricks/macros/adapters.sql | 6 ++ ...icks-dbt-constraints-vs-model-contracts.md | 98 +++++++++++++++++++ .../persist_constraints/models/schema.yml | 12 +++ .../models/table_model_contract.sql | 3 + .../test_persist_constraints.py | 25 +++++ 6 files changed, 147 insertions(+) create mode 100644 docs/databricks-dbt-constraints-vs-model-contracts.md create mode 100644 tests/integration/persist_constraints/models/table_model_contract.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 7934a30b..855c8246 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## dbt-databricks 1.5.0 (Release TBD) +### Under the hood +Throw an error if a model has an enforced contract. ([#322](https://github.com/databricks/dbt-databricks/pull/322)) + ## dbt-databricks 1.4.2 (February 17, 2023) ### Fixes diff --git a/dbt/include/databricks/macros/adapters.sql b/dbt/include/databricks/macros/adapters.sql index a3f812f7..d8d6b7ec 100644 --- a/dbt/include/databricks/macros/adapters.sql +++ b/dbt/include/databricks/macros/adapters.sql @@ -105,6 +105,12 @@ {% endmacro %} {% macro databricks__persist_constraints(relation, model) %} + {# Model contracts are not currently supported. #} + {%- set contract_config = config.get('contract') -%} + {% if contract_config and contract_config.enforced %} + {{ exceptions.raise_compiler_error('Model contracts are not currently supported.') }} + {% endif %} + {% if config.get('persist_constraints', False) and config.get('file_format', 'delta') == 'delta' %} {% do alter_table_add_constraints(relation, model.meta.constraints) %} {% do alter_column_set_constraints(relation, model.columns) %} diff --git a/docs/databricks-dbt-constraints-vs-model-contracts.md b/docs/databricks-dbt-constraints-vs-model-contracts.md new file mode 100644 index 00000000..df50bf52 --- /dev/null +++ b/docs/databricks-dbt-constraints-vs-model-contracts.md @@ -0,0 +1,98 @@ +# dbt-databricks constraints vs DBT 1.5 model contracts + +dbt-databricks constraints are enabled for a model by setting `persist_constraints: true` in the model configuration. Model contracts are enabled by setting `enforced: true` under the contract configuration. + +``` +models: + - name: table_model + config: + contract: + enforced: true +``` + +DBT model contracts enforce column names and datatypes. This means that **all** columns must be explicitly listed and have name and data_type properties. + +dbt-databricks constraints list model level constraints under `meta: constraints:` while in DBT `constraints` is a property of the model. +``` +dbt-databricks + +models: + - name: incremental_model + meta: + constraints: + - name: id_greater_than_zero + condition: id > 0 +``` + +``` +model contract + +models: + - name: table_model + constraints: + - type: check + columns: [id] + name: id_greater_than_zero + expression: "id > 0" +``` + +dbt-databricks constraints have a single column level constraint (currently limited to not_null) defined by the `meta: constraint:` property. +Model contracts have multiple column constraints listed under a columns `constraints` property. +``` +dbt-databricks + + columns: + - name: name + meta: + constraint: not_null +``` + +``` +model contract + + columns: + - name: name + data_type: string + constraints: + - type: not_null +``` + +Model contract constraint structure: +- **type** (required): dbt-databricks constraints do not have this property. DBT has not_null, check, primary_key, foreign_key, and custom types. dbt-databricks constraints currently support the equivalents of not_null and check. +- **expression**: Free text input to qualify the constraint. In dbt-databricks constraints this is the condition property. Note: in model contracts the expression text is contained by double quotes, the condition text in dbt-databricks constraints is not double quoted. +- **name** (optional in model contracts, required for check constraints in dbt-databricks constraints): Human-friendly name for this constraint. +- **columns** (model-level only): List of column names to apply the constraint over. dbt-databricks constraints do not have this property. + + + In a model contract a check constraint over a single column can be defined at either the model or the column level, but it is recommended that it be defined at the column level. Check constraints over multiple columns must be defined at the model level. +dbt-databricks check constraints are defined only at the model level. + +``` +dbt-databricks + +models: + - name: my_model + meta: + constraints: + - name: id_greater_than_zero + condition: id > 0 + columns: + - name: name + meta: + constraint: not_null +``` + +``` +model contract + +models: + - name: my_model + columns: + - name: name + data_type: integer + constraints: + - type: not_null + - type: check + name: id_greater_than_zero + expression: "id > 0" +``` \ No newline at end of file diff --git a/tests/integration/persist_constraints/models/schema.yml b/tests/integration/persist_constraints/models/schema.yml index 8c22905a..8a3de2ed 100644 --- a/tests/integration/persist_constraints/models/schema.yml +++ b/tests/integration/persist_constraints/models/schema.yml @@ -48,3 +48,15 @@ models: meta: constraint: not_null - name: date + + - name: table_model_contract + config: + contract: + enforced: true + columns: + - name: id + data_type: int + - name: name + data_type: string + - name: date + data_type: date \ No newline at end of file diff --git a/tests/integration/persist_constraints/models/table_model_contract.sql b/tests/integration/persist_constraints/models/table_model_contract.sql new file mode 100644 index 00000000..5f8e9141 --- /dev/null +++ b/tests/integration/persist_constraints/models/table_model_contract.sql @@ -0,0 +1,3 @@ +{{config(materialized='table')}} + +select * from {{ ref('seed') }} diff --git a/tests/integration/persist_constraints/test_persist_constraints.py b/tests/integration/persist_constraints/test_persist_constraints.py index 69399f5f..5bdabc6e 100644 --- a/tests/integration/persist_constraints/test_persist_constraints.py +++ b/tests/integration/persist_constraints/test_persist_constraints.py @@ -259,3 +259,28 @@ def test_databricks_sql_endpoint(self): @use_profile("databricks_uc_sql_endpoint") def test_databricks_uc_sql_endpoint(self): self.test_delta_constraints_disabled() + + +class TestModelContractNotSupported(TestConstraints): + def test_model_contract_not_supported(self): + model_name = "table_model_contract" + self.run_dbt(["seed"]) + self.run_and_check_failure( + model_name, err_msg="Model contracts are not currently supported." + ) + + @use_profile("databricks_cluster") + def test_databricks_cluster(self): + self.test_model_contract_not_supported() + + @use_profile("databricks_uc_cluster") + def test_databricks_uc_cluster(self): + self.test_model_contract_not_supported() + + @use_profile("databricks_sql_endpoint") + def test_databricks_sql_endpoint(self): + self.test_model_contract_not_supported() + + @use_profile("databricks_uc_sql_endpoint") + def test_databricks_uc_sql_endpoint(self): + self.test_model_contract_not_supported()