From e68b95c97a1acea8d4ea6463ad7312c3b2b8d70b Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 6 Jan 2021 03:37:27 -0800 Subject: [PATCH 01/29] dbt-utils macros now support dispatch --- macros/dbt_utils/schema_tests/at_least_one.sql | 2 +- macros/dbt_utils/schema_tests/cardinality_equality.sql | 2 +- macros/dbt_utils/schema_tests/expression_is_true.sql | 2 +- macros/dbt_utils/schema_tests/not_constant.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/macros/dbt_utils/schema_tests/at_least_one.sql b/macros/dbt_utils/schema_tests/at_least_one.sql index 670e17b..fa37949 100644 --- a/macros/dbt_utils/schema_tests/at_least_one.sql +++ b/macros/dbt_utils/schema_tests/at_least_one.sql @@ -1,4 +1,4 @@ -{% macro test_at_least_one(model) %} +{% macro sqlserver__test_at_least_one(model) %} {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} diff --git a/macros/dbt_utils/schema_tests/cardinality_equality.sql b/macros/dbt_utils/schema_tests/cardinality_equality.sql index 60d7a1b..dd5356d 100644 --- a/macros/dbt_utils/schema_tests/cardinality_equality.sql +++ b/macros/dbt_utils/schema_tests/cardinality_equality.sql @@ -1,4 +1,4 @@ -{% macro test_cardinality_equality(model, to, field) %} +{% macro sqlserver__test_cardinality_equality(model, to, field) %} {# T-SQL doesn't let you use numbers as aliases for columns #} {# Thus, no "GROUP BY 1" #} {% set column_name = kwargs.get('column_name', kwargs.get('from')) %} diff --git a/macros/dbt_utils/schema_tests/expression_is_true.sql b/macros/dbt_utils/schema_tests/expression_is_true.sql index e9da666..7213e73 100644 --- a/macros/dbt_utils/schema_tests/expression_is_true.sql +++ b/macros/dbt_utils/schema_tests/expression_is_true.sql @@ -1,4 +1,4 @@ -{% macro test_expression_is_true(model, condition='1=1') %} +{% macro sqlserver__test_expression_is_true(model, condition='1=1') %} {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} {# ref https://stackoverflow.com/a/7170753/3842610 #} {% set expression = kwargs.get('expression', kwargs.get('arg')) %} diff --git a/macros/dbt_utils/schema_tests/not_constant.sql b/macros/dbt_utils/schema_tests/not_constant.sql index 8ae9c12..0ad0298 100644 --- a/macros/dbt_utils/schema_tests/not_constant.sql +++ b/macros/dbt_utils/schema_tests/not_constant.sql @@ -1,5 +1,5 @@ -{% macro test_not_constant(model) %} +{% macro sqlserver__test_not_constant(model) %} {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} From 41547e23a9b216de46906a6d8a9ad056a0991f63 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 6 Jan 2021 04:52:18 -0800 Subject: [PATCH 02/29] fix this test as well --- .../mutually_exclusive_ranges.sql | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 macros/dbt_utils/schema_tests/mutually_exclusive_ranges.sql diff --git a/macros/dbt_utils/schema_tests/mutually_exclusive_ranges.sql b/macros/dbt_utils/schema_tests/mutually_exclusive_ranges.sql new file mode 100644 index 0000000..9adf9ad --- /dev/null +++ b/macros/dbt_utils/schema_tests/mutually_exclusive_ranges.sql @@ -0,0 +1,87 @@ +{% macro sqlserver__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed') %} + +{% if gaps == 'not_allowed' %} + {% set allow_gaps_operator='=' %} + {% set allow_gaps_operator_in_words='equal_to' %} +{% elif gaps == 'allowed' %} + {% set allow_gaps_operator='<=' %} + {% set allow_gaps_operator_in_words='less_than_or_equal_to' %} +{% elif gaps == 'required' %} + {% set allow_gaps_operator='<' %} + {% set allow_gaps_operator_in_words='less_than' %} +{% else %} + {{ exceptions.raise_compiler_error( + "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" + ) }} + +{% endif %} + +{% set partition_clause="partition by " ~ partition_by if partition_by else '' %} + +with window_functions as ( + + select + {% if partition_by %} + {{ partition_by }}, + {% endif %} + {{ lower_bound_column }} as lower_bound, + {{ upper_bound_column }} as upper_bound, + + lead({{ lower_bound_column }}) over ( + {{ partition_clause }} + order by {{ lower_bound_column }} + ) as next_lower_bound, + + case when + row_number() over ( + {{ partition_clause }} + order by {{ lower_bound_column }} desc + ) = 1 + then 1 else 0 end as is_last_record + from {{ model }} + +), + +calc as ( + -- We want to return records where one of our assumptions fails, so we'll use + -- the `not` function with `and` statements so we can write our assumptions nore cleanly + select + *, + + --TODO turn thesse into null ifs or case whens... + + -- For each record: lower_bound should be < upper_bound. + -- Coalesce it to return an error on the null case (implicit assumption + -- these columns are not_null) + coalesce( + lower_bound < upper_bound, + false + ) as lower_bound_less_than_upper_bound, + + -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. + -- Coalesce it to handle null cases for the last record. + coalesce( + upper_bound {{ allow_gaps_operator }} next_lower_bound, + is_last_record, + false + ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + + from window_functions + +), + +validation_errors as ( + + select + * + from calc + + where not( + -- THE FOLLOWING SHOULD BE TRUE -- + lower_bound_less_than_upper_bound + and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + ) +) + +select count(*) from validation_errors +{% endmacro %} From 4cfc406ca02853cb07ab0fa6dc4ddbb5ec2af92d Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 6 Jan 2021 23:15:40 -0800 Subject: [PATCH 03/29] TEMP move to theoretical branch --- .gitmodules | 3 ++- dbt-utils | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 8214c9d..9d90f9a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "dbt-utils"] path = dbt-utils - url = https://github.com/fishtown-analytics/dbt-utils + url = https://github.com/swanderz/dbt-utils + branch = tsql_compat diff --git a/dbt-utils b/dbt-utils index ebda584..2fbc2a0 160000 --- a/dbt-utils +++ b/dbt-utils @@ -1 +1 @@ -Subproject commit ebda5845df85194da981fa2c9da499750da51bd3 +Subproject commit 2fbc2a06759d52ea3e10d64b0f9c3fb2c87f9c50 From cb788d74ebb581fe1e8968c3d4a396b7a443047d Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 6 Jan 2021 23:33:37 -0800 Subject: [PATCH 04/29] shorten list of broken macros --- integration_tests/dbt_utils/dbt_project.yml | 55 +++++++++++---------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index eb2e8ac..c75622d 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -21,42 +21,43 @@ vars: models: dbt_utils_integration_tests: - +enabled: true - cross_db_utils: - test_dateadd: &disabled - +enabled: false - test_datediff: *disabled - test_hash: *disabled - test_last_day: *disabled - test_split_part: - +enabled: "{{ target.name != 'synapse' }}" +# +enabled: true +# cross_db_utils: +# test_dateadd: &disabled +# +enabled: false +# test_datediff: *disabled +# test_hash: *disabled +# test_last_day: *disabled +# test_split_part: +# +enabled: "{{ target.name != 'synapse' }}" datetime: - test_date_spine: *disabled - materializations: *disabled + test_date_spine: &disabled # BROKEN DUE TO MODEL DEF + +enabled: false +# materializations: *disabled - schema_tests: - data_test_mutually_exclusive_ranges_no_gaps: *disabled - data_test_mutually_exclusive_ranges_with_gaps: *disabled - data_test_not_constant: *disabled - data_test_relationships_where_table_2: *disabled - data_test_unique_where: *disabled - data_test_not_null_where: *disabled + # schema_tests: + # data_test_mutually_exclusive_ranges_no_gaps: *disabled + # data_test_mutually_exclusive_ranges_with_gaps: *disabled + # data_test_not_constant: *disabled + # data_test_relationships_where_table_2: *disabled + # data_test_unique_where: *disabled + # data_test_not_null_where: *disabled # the following work but only when # the dbt-utils submodule macros are overwritten - data_test_at_least_one: *disabled - data_people: *disabled - data_test_expression_is_true: *disabled - data_test_not_constant: *disabled - sql: + # data_test_at_least_one: *disabled + # data_people: *disabled + # data_test_expression_is_true: *disabled + # data_test_not_constant: *disabled + sql: # BROKEN DUE TO MODEL DEFS test_generate_series: *disabled test_get_column_values: *disabled test_get_relations_by_pattern: *disabled test_get_relations_by_prefix_and_union: *disabled test_groupby: *disabled - get_query_results_as_dict: *disabled - test_surrogate_key: *disabled - test_union: *disabled - web: + # get_query_results_as_dict: *disabled + # test_surrogate_key: *disabled + # test_union: *disabled + web: # BROKEN DUE TO TEST DEFS test_url_host: *disabled test_url_path: *disabled test_urls: *disabled From 6f205e36968571bea407fa9d809e8a2334e8f1b0 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 00:08:51 -0800 Subject: [PATCH 05/29] must be string for comparison --- macros/dbt_utils/cross_db_utils/hash.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/dbt_utils/cross_db_utils/hash.sql b/macros/dbt_utils/cross_db_utils/hash.sql index db5784b..dfe68e9 100644 --- a/macros/dbt_utils/cross_db_utils/hash.sql +++ b/macros/dbt_utils/cross_db_utils/hash.sql @@ -1,5 +1,5 @@ {% macro sqlserver__hash(field) %} - hashbytes('md5', {{field}}) + convert(varchar(50), hashbytes('md5', {{field}}), 2) {% endmacro %} From 96e29db10b4741745be0c20e5c8da2eff84bb3f5 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 00:12:24 -0800 Subject: [PATCH 06/29] synapse does not yet support --- integration_tests/dbt_utils/dbt_project.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index c75622d..99e285e 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -73,8 +73,9 @@ seeds: updated_at: datetime day: date month: date - data_split_part: + data_split_part: ¬-synapse +enabled: "{{ target.name != 'synapse' }}" + test_split_part: *not-synapse data_dateadd: +column_types: From f47fb18638d996bd082af78688ce4a41804b6834 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 00:12:52 -0800 Subject: [PATCH 07/29] fixed by dbt-util dispatchification --- integration_tests/dbt_utils/dbt_project.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index 99e285e..f172995 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -43,11 +43,6 @@ models: # data_test_unique_where: *disabled # data_test_not_null_where: *disabled # the following work but only when - # the dbt-utils submodule macros are overwritten - # data_test_at_least_one: *disabled - # data_people: *disabled - # data_test_expression_is_true: *disabled - # data_test_not_constant: *disabled sql: # BROKEN DUE TO MODEL DEFS test_generate_series: *disabled test_get_column_values: *disabled From 31151a088294d099b9ecc75cebb0ca2b2a377ab2 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 00:15:11 -0800 Subject: [PATCH 08/29] synapse does not support --- integration_tests/dbt_utils/dbt_project.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index f172995..eb8aa94 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -21,15 +21,15 @@ vars: models: dbt_utils_integration_tests: -# +enabled: true -# cross_db_utils: + +enabled: true + cross_db_utils: # test_dateadd: &disabled # +enabled: false # test_datediff: *disabled # test_hash: *disabled # test_last_day: *disabled -# test_split_part: -# +enabled: "{{ target.name != 'synapse' }}" + test_split_part: ¬-synapse + +enabled: "{{ target.name != 'synapse' }}" datetime: test_date_spine: &disabled # BROKEN DUE TO MODEL DEF +enabled: false @@ -68,9 +68,7 @@ seeds: updated_at: datetime day: date month: date - data_split_part: ¬-synapse - +enabled: "{{ target.name != 'synapse' }}" - test_split_part: *not-synapse + data_split_part: *not-synapse data_dateadd: +column_types: From 1aeb356f017e64c5b3a268a275d8a81ed45566be Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 00:27:55 -0800 Subject: [PATCH 09/29] clean up --- integration_tests/dbt_utils/dbt_project.yml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index eb8aa94..c10920f 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -23,35 +23,19 @@ models: dbt_utils_integration_tests: +enabled: true cross_db_utils: -# test_dateadd: &disabled -# +enabled: false -# test_datediff: *disabled -# test_hash: *disabled -# test_last_day: *disabled test_split_part: ¬-synapse +enabled: "{{ target.name != 'synapse' }}" datetime: test_date_spine: &disabled # BROKEN DUE TO MODEL DEF +enabled: false -# materializations: *disabled - - # schema_tests: - # data_test_mutually_exclusive_ranges_no_gaps: *disabled - # data_test_mutually_exclusive_ranges_with_gaps: *disabled - # data_test_not_constant: *disabled - # data_test_relationships_where_table_2: *disabled - # data_test_unique_where: *disabled - # data_test_not_null_where: *disabled - # the following work but only when + materializations: + test_insert_by_period: *disabled sql: # BROKEN DUE TO MODEL DEFS test_generate_series: *disabled test_get_column_values: *disabled test_get_relations_by_pattern: *disabled test_get_relations_by_prefix_and_union: *disabled test_groupby: *disabled - # get_query_results_as_dict: *disabled - # test_surrogate_key: *disabled - # test_union: *disabled web: # BROKEN DUE TO TEST DEFS test_url_host: *disabled test_url_path: *disabled From 72f422f35d19e367c770b0f5418b54e469168f2b Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 00:53:55 -0800 Subject: [PATCH 10/29] downstream fix --- .../dbt_utils/schema_tests/expression_is_true.sql | 12 +++++++++--- .../dbt_utils/schema_tests/relationships_where.sql | 13 +++++++++++++ .../dbt_utils/schema_tests/test_not_null_where.sql | 12 ++++++++++++ macros/dbt_utils/schema_tests/test_unique_where.sql | 12 ++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 macros/dbt_utils/schema_tests/relationships_where.sql create mode 100644 macros/dbt_utils/schema_tests/test_not_null_where.sql create mode 100644 macros/dbt_utils/schema_tests/test_unique_where.sql diff --git a/macros/dbt_utils/schema_tests/expression_is_true.sql b/macros/dbt_utils/schema_tests/expression_is_true.sql index 7213e73..04c01e2 100644 --- a/macros/dbt_utils/schema_tests/expression_is_true.sql +++ b/macros/dbt_utils/schema_tests/expression_is_true.sql @@ -1,6 +1,12 @@ -{% macro sqlserver__test_expression_is_true(model, condition='1=1') %} -{# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} -{# ref https://stackoverflow.com/a/7170753/3842610 #} +{% macro sqlserver__test_expression_is_true(model, condition) %} + + {# override generic default #} + {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} + {# ref https://stackoverflow.com/a/7170753/3842610 #} + {% if condition == 'true' %} + {% set condition = '1=1' %} + {% endif %} + {% set expression = kwargs.get('expression', kwargs.get('arg')) %} with meet_condition as ( diff --git a/macros/dbt_utils/schema_tests/relationships_where.sql b/macros/dbt_utils/schema_tests/relationships_where.sql new file mode 100644 index 0000000..5108aad --- /dev/null +++ b/macros/dbt_utils/schema_tests/relationships_where.sql @@ -0,0 +1,13 @@ +{% macro sqlserver__test_relationships_where(model) %} + + {% set from_condition = kwargs.get('from_condition', "1=1") %} + {% set to_condition = kwargs.get('to_condition', "1=1") %} + {# override generic default #} + {# TSQL has non-ANSI not-equal sign #} + {% if from_condition == 'id <> 4' %} + {% set where = 'id != 4' %} + {% endif %} + + {{ return(default__test_not_null_where(model)) }} + +{% endmacro %} diff --git a/macros/dbt_utils/schema_tests/test_not_null_where.sql b/macros/dbt_utils/schema_tests/test_not_null_where.sql new file mode 100644 index 0000000..62c8c16 --- /dev/null +++ b/macros/dbt_utils/schema_tests/test_not_null_where.sql @@ -0,0 +1,12 @@ +{% macro sqlserver__test_not_null_where(model) %} + + {% set where = kwargs.get('where', kwargs.get('arg')) %} + {# override generic default #} + {# TSQL has no bool type #} + {% if where == '_deleted = false' %} + {% set where = '_deleted = 0' %} + {% endif %} + + {{ return(default__test_not_null_where(model)) }} + +{% endmacro %} diff --git a/macros/dbt_utils/schema_tests/test_unique_where.sql b/macros/dbt_utils/schema_tests/test_unique_where.sql new file mode 100644 index 0000000..f79163d --- /dev/null +++ b/macros/dbt_utils/schema_tests/test_unique_where.sql @@ -0,0 +1,12 @@ +{% macro sqlserver__test_unique_where(model) %} + + {% set where = kwargs.get('where', kwargs.get('arg')) %} + {# override generic default #} + {# TSQL has no bool type #} + {% if where == '_deleted = false' %} + {% set where = '_deleted = 0' %} + {% endif %} + + {{ return(default__test_unique_where(model)) }} + +{% endmacro %} From a37e6c87cefd638db7559b5ea497b0f1f3c626c6 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 01:03:44 -0800 Subject: [PATCH 11/29] update submodule --- dbt-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-utils b/dbt-utils index 2fbc2a0..c576b75 160000 --- a/dbt-utils +++ b/dbt-utils @@ -1 +1 @@ -Subproject commit 2fbc2a06759d52ea3e10d64b0f9c3fb2c87f9c50 +Subproject commit c576b75aad6df81aa20518269f4ee315b715d589 From 47711defa60a2c3f5fc6ef84a9abbadb7aab65e3 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 01:26:49 -0800 Subject: [PATCH 12/29] WHY PYTHON WHY!!! --- macros/dbt_utils/schema_tests/relationships_where.sql | 5 +++-- macros/dbt_utils/schema_tests/test_not_null_where.sql | 3 ++- macros/dbt_utils/schema_tests/test_unique_where.sql | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/macros/dbt_utils/schema_tests/relationships_where.sql b/macros/dbt_utils/schema_tests/relationships_where.sql index 5108aad..bd5ee19 100644 --- a/macros/dbt_utils/schema_tests/relationships_where.sql +++ b/macros/dbt_utils/schema_tests/relationships_where.sql @@ -1,5 +1,6 @@ -{% macro sqlserver__test_relationships_where(model) %} +{% macro sqlserver__test_relationships_where(model, to, field) %} + {% set column_name = kwargs.get('column_name', kwargs.get('from')) %} {% set from_condition = kwargs.get('from_condition', "1=1") %} {% set to_condition = kwargs.get('to_condition', "1=1") %} {# override generic default #} @@ -8,6 +9,6 @@ {% set where = 'id != 4' %} {% endif %} - {{ return(default__test_not_null_where(model)) }} + {{ return(dbt_utils.default__test_relationships_where(model, to, field, column_name=column_name, from_condition=from_condition, to_condition=to_condition)) }} {% endmacro %} diff --git a/macros/dbt_utils/schema_tests/test_not_null_where.sql b/macros/dbt_utils/schema_tests/test_not_null_where.sql index 62c8c16..d0fa246 100644 --- a/macros/dbt_utils/schema_tests/test_not_null_where.sql +++ b/macros/dbt_utils/schema_tests/test_not_null_where.sql @@ -1,5 +1,6 @@ {% macro sqlserver__test_not_null_where(model) %} + {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} {% set where = kwargs.get('where', kwargs.get('arg')) %} {# override generic default #} {# TSQL has no bool type #} @@ -7,6 +8,6 @@ {% set where = '_deleted = 0' %} {% endif %} - {{ return(default__test_not_null_where(model)) }} + {{ return(dbt_utils.default__test_not_null_where(model, column_name=column_name, where=where)) }} {% endmacro %} diff --git a/macros/dbt_utils/schema_tests/test_unique_where.sql b/macros/dbt_utils/schema_tests/test_unique_where.sql index f79163d..b33e146 100644 --- a/macros/dbt_utils/schema_tests/test_unique_where.sql +++ b/macros/dbt_utils/schema_tests/test_unique_where.sql @@ -1,5 +1,5 @@ {% macro sqlserver__test_unique_where(model) %} - + {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} {% set where = kwargs.get('where', kwargs.get('arg')) %} {# override generic default #} {# TSQL has no bool type #} @@ -7,6 +7,6 @@ {% set where = '_deleted = 0' %} {% endif %} - {{ return(default__test_unique_where(model)) }} + {{ return(dbt_utils.default__test_unique_where(model, column_name=column_name, where=where)) }} {% endmacro %} From 9c43f6bfa7a152ad16ea5fefb3d562243badc331 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 17:35:55 -0800 Subject: [PATCH 13/29] inform users why --- integration_tests/dbt_utils/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index c10920f..d166e25 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -35,7 +35,7 @@ models: test_get_column_values: *disabled test_get_relations_by_pattern: *disabled test_get_relations_by_prefix_and_union: *disabled - test_groupby: *disabled + test_groupby: *disabled # TSQL doesn't let you group by column numbers web: # BROKEN DUE TO TEST DEFS test_url_host: *disabled test_url_path: *disabled From 4230be2d023e3dba8f75c6fb1d0b35b7c1a2715a Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 21:51:45 -0800 Subject: [PATCH 14/29] disable until #18 is finished --- integration_tests/dbt_utils/dbt_project.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index d166e25..8bf2ee1 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -45,7 +45,9 @@ seeds: +quote_columns: false dbt_utils_integration_tests: - + schema_tests: + data_test_mutually_exclusive_ranges_no_gaps: *disabled + data_test_mutually_exclusive_ranges_with_gaps: *disabled cross_db: data_date_trunc: +column_types: From 008ba9c080ba0bb118ecaf423a28407c51b79c67 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 22:22:46 -0800 Subject: [PATCH 15/29] unsupported --- integration_tests/dbt_utils/dbt_project.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index 8bf2ee1..e87a114 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -25,6 +25,9 @@ models: cross_db_utils: test_split_part: ¬-synapse +enabled: "{{ target.name != 'synapse' }}" + # Synapse does not support timestamp datatype see: + # https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-tables-data-types#unsupported-data-types + test_dateadd: *not-synapse datetime: test_date_spine: &disabled # BROKEN DUE TO MODEL DEF +enabled: false From 270e953e5aa54b0d7a8fbe697e614a95e7f63704 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 7 Jan 2021 23:26:31 -0800 Subject: [PATCH 16/29] synapse does not support timestamp --- integration_tests/dbt_utils/dbt_project.yml | 3 --- macros/dbt_utils/cross_db_utils/datatypes.sql | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index e87a114..8bf2ee1 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -25,9 +25,6 @@ models: cross_db_utils: test_split_part: ¬-synapse +enabled: "{{ target.name != 'synapse' }}" - # Synapse does not support timestamp datatype see: - # https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-tables-data-types#unsupported-data-types - test_dateadd: *not-synapse datetime: test_date_spine: &disabled # BROKEN DUE TO MODEL DEF +enabled: false diff --git a/macros/dbt_utils/cross_db_utils/datatypes.sql b/macros/dbt_utils/cross_db_utils/datatypes.sql index a534a85..0a6596a 100644 --- a/macros/dbt_utils/cross_db_utils/datatypes.sql +++ b/macros/dbt_utils/cross_db_utils/datatypes.sql @@ -2,6 +2,13 @@ VARCHAR {%- endmacro -%} +-- TEMP UNTIL synapse is standalone adapter type +{% macro sqlserver__type_timestamp() %} + {# Synapse does not support timestamp datatype see: #} + {# https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-tables-data-types#unsupported-data-types #} + datetime +{% endmacro %} + {# Imagine an adapter plugin, dbt-synapse, that inherits from dbt-sqlserver. For the time being, we need to explicitly reimplement sqlserver__ macros @@ -14,3 +21,10 @@ {% macro synapse__type_string(field) %} {% do return(sqlserver__type_string()) %} {% endmacro %} + + +{% macro synapse__type_timestamp() %} + {# Synapse does not support timestamp datatype see: #} + {# https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-tables-data-types#unsupported-data-types #} + datetime +{% endmacro %} \ No newline at end of file From 37449a994bb47f5d5e894cb0bf4646b9cbb06812 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Fri, 8 Jan 2021 01:48:44 -0800 Subject: [PATCH 17/29] replacement for 'limit 0' --- integration_tests/dbt_utils/macros/limit_zero.sql | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 integration_tests/dbt_utils/macros/limit_zero.sql diff --git a/integration_tests/dbt_utils/macros/limit_zero.sql b/integration_tests/dbt_utils/macros/limit_zero.sql new file mode 100644 index 0000000..cf66cc6 --- /dev/null +++ b/integration_tests/dbt_utils/macros/limit_zero.sql @@ -0,0 +1,3 @@ +{% macro sqlserver__limit_zero() %} + {{ return('where 0=1') }} +{% endmacro %} \ No newline at end of file From 50811e81206ca0454a3deb6dbd985dc536365390 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Fri, 8 Jan 2021 02:19:55 -0800 Subject: [PATCH 18/29] hail mary --- integration_tests/dbt_utils/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index 8bf2ee1..f0b4834 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -17,7 +17,7 @@ clean-targets: # directories to be removed by `dbt clean` - "dbt_modules" vars: - dbt_utils_dispatch_list: ['tsql_utils'] + dbt_utils_dispatch_list: ['tsql_utils', 'dbt_utils'] models: dbt_utils_integration_tests: From 88a4beb0ec25434c786374c7202bb661a7828a48 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Fri, 8 Jan 2021 11:38:39 -0800 Subject: [PATCH 19/29] add limit zero to dispatch namespace --- integration_tests/dbt_utils/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index f0b4834..c55fe4f 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -17,7 +17,7 @@ clean-targets: # directories to be removed by `dbt clean` - "dbt_modules" vars: - dbt_utils_dispatch_list: ['tsql_utils', 'dbt_utils'] + dbt_utils_dispatch_list: ['tsql_utils', 'tsql_utils_dbt_utils_integration_tests'] models: dbt_utils_integration_tests: From 80ca9b7b91ca25a450f17e5dea0c6129b02c70c2 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Fri, 8 Jan 2021 11:54:31 -0800 Subject: [PATCH 20/29] disabled due to other issue --- integration_tests/dbt_utils/dbt_project.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index c55fe4f..de04abe 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -30,12 +30,19 @@ models: +enabled: false materializations: test_insert_by_period: *disabled - sql: # BROKEN DUE TO MODEL DEFS + sql: # BROKEN BC MACROS ARE UNPORTED STILL test_generate_series: *disabled test_get_column_values: *disabled test_get_relations_by_pattern: *disabled test_get_relations_by_prefix_and_union: *disabled test_groupby: *disabled # TSQL doesn't let you group by column numbers + # BROKEN DUE TO SYNAPSE SEED LOADING EMPTY CSV VALS AS EMPTY STRINGS + # see: https://github.com/dbt-msft/dbt-synapse/issues/36 + test_dateadd: *not-synapse + test_datediff: *not-synapse + test_hash: *not-synapse + test_last_day: *not-synapse + test_union: *not-synapse web: # BROKEN DUE TO TEST DEFS test_url_host: *disabled test_url_path: *disabled From fff18d3dc5375167b328bd03521f7bd9d9c777b8 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Sun, 10 Jan 2021 11:58:56 -0800 Subject: [PATCH 21/29] correct macro directory --- integration_tests/dbt_utils/dbt_project.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index de04abe..ebd0269 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -25,6 +25,13 @@ models: cross_db_utils: test_split_part: ¬-synapse +enabled: "{{ target.name != 'synapse' }}" + # BROKEN DUE TO SYNAPSE SEED LOADING EMPTY CSV VALS AS EMPTY STRINGS + # see: https://github.com/dbt-msft/dbt-synapse/issues/36 + test_dateadd: *not-synapse + test_datediff: *not-synapse + test_hash: *not-synapse + test_last_day: *not-synapse + test_union: *not-synapse datetime: test_date_spine: &disabled # BROKEN DUE TO MODEL DEF +enabled: false @@ -36,13 +43,6 @@ models: test_get_relations_by_pattern: *disabled test_get_relations_by_prefix_and_union: *disabled test_groupby: *disabled # TSQL doesn't let you group by column numbers - # BROKEN DUE TO SYNAPSE SEED LOADING EMPTY CSV VALS AS EMPTY STRINGS - # see: https://github.com/dbt-msft/dbt-synapse/issues/36 - test_dateadd: *not-synapse - test_datediff: *not-synapse - test_hash: *not-synapse - test_last_day: *not-synapse - test_union: *not-synapse web: # BROKEN DUE TO TEST DEFS test_url_host: *disabled test_url_path: *disabled From 74f9d84792ca56d77d4b4deef0d3a611e9c5906e Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Sun, 10 Jan 2021 21:17:22 -0800 Subject: [PATCH 22/29] wrong section --- integration_tests/dbt_utils/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index ebd0269..6c96b54 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -31,7 +31,6 @@ models: test_datediff: *not-synapse test_hash: *not-synapse test_last_day: *not-synapse - test_union: *not-synapse datetime: test_date_spine: &disabled # BROKEN DUE TO MODEL DEF +enabled: false @@ -43,6 +42,7 @@ models: test_get_relations_by_pattern: *disabled test_get_relations_by_prefix_and_union: *disabled test_groupby: *disabled # TSQL doesn't let you group by column numbers + test_union: *not-synapse web: # BROKEN DUE TO TEST DEFS test_url_host: *disabled test_url_path: *disabled From 6eb5af326f1182d99d04b259b3bffa3a1e0116d4 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Mon, 11 Jan 2021 08:27:01 -0800 Subject: [PATCH 23/29] changes are in dbt-utils now! --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9d90f9a..62889bf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "dbt-utils"] path = dbt-utils - url = https://github.com/swanderz/dbt-utils - branch = tsql_compat + url = https://github.com/fishtown-analytics/dbt-utils + branch = master From a274666f3d76dd280eee9dcd677ac8971e270632 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Mon, 11 Jan 2021 08:31:57 -0800 Subject: [PATCH 24/29] pull from master w/ new PRs merged --- dbt-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-utils b/dbt-utils index c576b75..bbba960 160000 --- a/dbt-utils +++ b/dbt-utils @@ -1 +1 @@ -Subproject commit c576b75aad6df81aa20518269f4ee315b715d589 +Subproject commit bbba960726667abc66b42624f0d36bbb62c37593 From 04c9234def1d3c57d98db0fcf64ceb499e4dbb23 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Mon, 11 Jan 2021 08:41:12 -0800 Subject: [PATCH 25/29] perhaps these work now --- integration_tests/dbt_utils/dbt_project.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index 6c96b54..e8534d7 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -37,10 +37,10 @@ models: materializations: test_insert_by_period: *disabled sql: # BROKEN BC MACROS ARE UNPORTED STILL - test_generate_series: *disabled - test_get_column_values: *disabled - test_get_relations_by_pattern: *disabled - test_get_relations_by_prefix_and_union: *disabled + # test_generate_series: *disabled + # test_get_column_values: *disabled + # test_get_relations_by_pattern: *disabled + # test_get_relations_by_prefix_and_union: *disabled test_groupby: *disabled # TSQL doesn't let you group by column numbers test_union: *not-synapse web: # BROKEN DUE TO TEST DEFS From f46b79336ca54446fd50ccf531252222151992ee Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Mon, 11 Jan 2021 08:45:57 -0800 Subject: [PATCH 26/29] still unported --- integration_tests/dbt_utils/dbt_project.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/dbt_utils/dbt_project.yml b/integration_tests/dbt_utils/dbt_project.yml index e8534d7..6c96b54 100644 --- a/integration_tests/dbt_utils/dbt_project.yml +++ b/integration_tests/dbt_utils/dbt_project.yml @@ -37,10 +37,10 @@ models: materializations: test_insert_by_period: *disabled sql: # BROKEN BC MACROS ARE UNPORTED STILL - # test_generate_series: *disabled - # test_get_column_values: *disabled - # test_get_relations_by_pattern: *disabled - # test_get_relations_by_prefix_and_union: *disabled + test_generate_series: *disabled + test_get_column_values: *disabled + test_get_relations_by_pattern: *disabled + test_get_relations_by_prefix_and_union: *disabled test_groupby: *disabled # TSQL doesn't let you group by column numbers test_union: *not-synapse web: # BROKEN DUE TO TEST DEFS From 0217691e1efd0a3fbbb72364afbd117975b6d167 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Mon, 11 Jan 2021 23:06:47 -0800 Subject: [PATCH 27/29] dbt-utils default arg is now TSQL compatible --- macros/dbt_utils/schema_tests/expression_is_true.sql | 7 ------- 1 file changed, 7 deletions(-) diff --git a/macros/dbt_utils/schema_tests/expression_is_true.sql b/macros/dbt_utils/schema_tests/expression_is_true.sql index 470e78e..578fda4 100644 --- a/macros/dbt_utils/schema_tests/expression_is_true.sql +++ b/macros/dbt_utils/schema_tests/expression_is_true.sql @@ -1,12 +1,5 @@ {% macro sqlserver__test_expression_is_true(model, condition) %} - {# override generic default #} - {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} - {# ref https://stackoverflow.com/a/7170753/3842610 #} - {% if condition == 'true' %} - {% set condition = '1=1' %} - {% endif %} - {% set expression = kwargs.get('expression', kwargs.get('arg')) %} with meet_condition as ( From 49c41a380973b31d5f681598dfac6c44dbf7c0da Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 13 Jan 2021 16:19:01 -0800 Subject: [PATCH 28/29] more context --- macros/dbt_utils/schema_tests/relationships_where.sql | 3 ++- macros/dbt_utils/schema_tests/test_not_null_where.sql | 3 ++- macros/dbt_utils/schema_tests/test_unique_where.sql | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/macros/dbt_utils/schema_tests/relationships_where.sql b/macros/dbt_utils/schema_tests/relationships_where.sql index bd5ee19..5143085 100644 --- a/macros/dbt_utils/schema_tests/relationships_where.sql +++ b/macros/dbt_utils/schema_tests/relationships_where.sql @@ -3,7 +3,8 @@ {% set column_name = kwargs.get('column_name', kwargs.get('from')) %} {% set from_condition = kwargs.get('from_condition', "1=1") %} {% set to_condition = kwargs.get('to_condition', "1=1") %} - {# override generic default #} + {# override dbt-utils' integration tests args default see: #} + {# https://github.com/fishtown-analytics/dbt-utils/blob/bbba960726667abc66b42624f0d36bbb62c37593/integration_tests/models/schema_tests/schema.yml#L67-L75 #} {# TSQL has non-ANSI not-equal sign #} {% if from_condition == 'id <> 4' %} {% set where = 'id != 4' %} diff --git a/macros/dbt_utils/schema_tests/test_not_null_where.sql b/macros/dbt_utils/schema_tests/test_not_null_where.sql index d0fa246..a2e87ab 100644 --- a/macros/dbt_utils/schema_tests/test_not_null_where.sql +++ b/macros/dbt_utils/schema_tests/test_not_null_where.sql @@ -2,7 +2,8 @@ {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} {% set where = kwargs.get('where', kwargs.get('arg')) %} - {# override generic default #} + {# override dbt-utils' integration tests args default see: #} + {# https://github.com/fishtown-analytics/dbt-utils/blob/bbba960726667abc66b42624f0d36bbb62c37593/integration_tests/models/schema_tests/schema.yml#L53-L65 #} {# TSQL has no bool type #} {% if where == '_deleted = false' %} {% set where = '_deleted = 0' %} diff --git a/macros/dbt_utils/schema_tests/test_unique_where.sql b/macros/dbt_utils/schema_tests/test_unique_where.sql index b33e146..4f4aa73 100644 --- a/macros/dbt_utils/schema_tests/test_unique_where.sql +++ b/macros/dbt_utils/schema_tests/test_unique_where.sql @@ -1,7 +1,8 @@ {% macro sqlserver__test_unique_where(model) %} {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} {% set where = kwargs.get('where', kwargs.get('arg')) %} - {# override generic default #} + {# override dbt-utils' integration tests args default see: #} + {# https://github.com/fishtown-analytics/dbt-utils/blob/bbba960726667abc66b42624f0d36bbb62c37593/integration_tests/models/schema_tests/schema.yml#L53-L65 #} {# TSQL has no bool type #} {% if where == '_deleted = false' %} {% set where = '_deleted = 0' %} From 9449055451678f42f73d9d41e5e31e791a62b41e Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 13 Jan 2021 16:29:01 -0800 Subject: [PATCH 29/29] pin to latest release --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21441b3..ad9162a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,7 +61,7 @@ jobs: python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools - pip install git+https://github.com/dbt-msft/dbt-synapse.git + pip install dbt-synapse==0.18.1 mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml