Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dispatch-ify ALL THE MACROS! #312

Merged
merged 30 commits into from
Jan 10, 2021
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
844bfa1
dispatch-ify schema tests
dataders Dec 23, 2020
16bd69a
typo
dataders Dec 23, 2020
fe36124
generalized macros
chaerinlee1 Jan 4, 2021
3e886be
generalized macros progress
chaerinlee1 Jan 4, 2021
730f7a1
remove prefix from tests
dataders Jan 4, 2021
21be8a4
add keyword args to adapter.disptach() arg list
dataders Jan 4, 2021
d35737c
HAIL MARY find logger macros
dataders Jan 4, 2021
1c6ff84
for discovery in user projects & other pkgs
dataders Jan 4, 2021
61d0dd1
for discovery in user projects & other pkgs
dataders Jan 4, 2021
ae8d040
generalize rest of macros
chaerinlee1 Jan 5, 2021
e3417af
datatype bug fix
chaerinlee1 Jan 5, 2021
b3428bc
datatype bug fix to all macros
chaerinlee1 Jan 5, 2021
6893304
compilation error bug fix
chaerinlee1 Jan 5, 2021
274a09b
undo last commit
chaerinlee1 Jan 5, 2021
832f2ef
dispatch shouldn't override macro signature
dataders Jan 5, 2021
f940504
macro uses varargs
dataders Jan 5, 2021
cd40119
missing close paren
dataders Jan 5, 2021
b4aa854
macro uses varargs
dataders Jan 5, 2021
b5a14ef
hail mary
dataders Jan 5, 2021
c9cfad1
whisper Jinja sweet nothings
dataders Jan 5, 2021
39775c6
sweet hail mary nothings
dataders Jan 5, 2021
278514b
give the Jinja what it wants
dataders Jan 5, 2021
9bd577b
goodnight sweet Jinja
dataders Jan 5, 2021
bb3bf4e
keep dispatch from overriding args
dataders Jan 5, 2021
66f0bb0
stop dispatch from overridding args
dataders Jan 5, 2021
7406f45
removed default args
chaerinlee1 Jan 5, 2021
2cb81ff
undo small tweak
dataders Jan 5, 2021
178cdbb
not needed
dataders Jan 6, 2021
e4004c7
Merge branch 'master' of https://github.com/fishtown-analytics/dbt-ut…
dataders Jan 8, 2021
20034bd
for posterity
dataders Jan 8, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions integration_tests/dbt_project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ clean-targets: # directories to be removed by `dbt clean`
- "target"
- "dbt_modules"

vars:
dbt_utils_dispatch_list: ['dbt_utils']

dataders marked this conversation as resolved.
Show resolved Hide resolved
seeds:

+quote_columns: false
Expand Down
7 changes: 7 additions & 0 deletions macros/datetime/date_spine.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
{% macro get_intervals_between(start_date, end_date, datepart) -%}
{{ adapter.dispatch('get_intervals_between', packages = dbt_utils._get_utils_namespaces())(start_date, end_date, datepart) }}
{%- endmacro %}

{% macro default__get_intervals_between(start_date, end_date, datepart) -%}
{%- call statement('get_intervals_between', fetch_result=True) %}

select {{dbt_utils.datediff(start_date, end_date, datepart)}}
Expand All @@ -21,6 +24,10 @@


{% macro date_spine(datepart, start_date, end_date) %}
{{ adapter.dispatch('date_spine', packages = dbt_utils._get_utils_namespaces())(datepart, start_date, end_date) }}
{%- endmacro %}

{% macro default__date_spine(datepart, start_date, end_date) %}

/*
call as follows:
Expand Down
4 changes: 4 additions & 0 deletions macros/geo/haversine_distance.sql
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ The arguments should be float type.
#}

{% macro haversine_distance(lat1,lon1,lat2,lon2) -%}
{{ adapter.dispatch('haversine_distance', packages = dbt_utils._get_utils_namespaces())(lat1,lon1,lat2,lon2) }}
{% endmacro %}

{% macro default__haversine_distance(lat1,lon1,lat2,lon2) -%}

2 * 3961 * asin(sqrt((sin(radians(({{lat2}} - {{lat1}}) / 2))) ^ 2 +
cos(radians({{lat1}})) * cos(radians({{lat2}})) *
Expand Down
4 changes: 3 additions & 1 deletion macros/logger/log_info.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{% macro log_info(message) %}
{{ adapter.dispatch('log_info', packages = dbt_utils._get_utils_namespaces())(message) }}
{% endmacro %}

{% macro default__log_info(message) %}
{{ log(dbt_utils.pretty_log_format(message), info=True) }}

{% endmacro %}
4 changes: 3 additions & 1 deletion macros/logger/pretty_log_format.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{% macro pretty_log_format(message) %}
{{ adapter.dispatch('pretty_log_format', packages = dbt_utils._get_utils_namespaces())(message) }}
{% endmacro %}

{% macro default__pretty_log_format(message) %}
{{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }}

{% endmacro %}
4 changes: 3 additions & 1 deletion macros/logger/pretty_time.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
{% macro pretty_time(format='%H:%M:%S') %}
{{ adapter.dispatch('pretty_time', packages = dbt_utils._get_utils_namespaces())(format='%H:%M:%S') }}
{% endmacro %}

{% macro default__pretty_time(format='%H:%M:%S') %}
{{ return(modules.datetime.datetime.now().strftime(format)) }}

{% endmacro %}
4 changes: 4 additions & 0 deletions macros/materializations/insert_by_period_materialization.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}
{{ adapter.dispatch('get_period_boundaries', packages = dbt_utils._get_utils_namespaces())(target_schema, target_table, timestamp_field, start_date, stop_date, period) }}
{% endmacro %}

{% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}

{% call statement('period_boundaries', fetch_result=True) -%}
with data as (
Expand Down
6 changes: 5 additions & 1 deletion macros/schema_tests/at_least_one.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_at_least_one(model) %}
{{ adapter.dispatch('test_at_least_one', packages = dbt_utils._get_utils_namespaces())(model, **kwargs) }}
{% endmacro %}

{% macro default__test_at_least_one(model) %}

{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}

Expand All @@ -14,4 +18,4 @@ from (

) validation_errors

{% endmacro %}
{% endmacro %}
5 changes: 5 additions & 0 deletions macros/schema_tests/cardinality_equality.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
{% macro test_cardinality_equality(model, to, field) %}
{{ adapter.dispatch('test_cardinality_equality', packages = dbt_utils._get_utils_namespaces())(model, to, field, **kwargs) }}

{% endmacro %}

{% macro default__test_cardinality_equality(model, to, field) %}

{% set column_name = kwargs.get('column_name', kwargs.get('from')) %}

Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/equal_rowcount.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_equal_rowcount(model) %}
{{ adapter.dispatch('test_equal_rowcount', packages = dbt_utils._get_utils_namespaces())(model, **kwargs) }}
{% endmacro %}

{% macro default__test_equal_rowcount(model) %}

{% set compare_model = kwargs.get('compare_model', kwargs.get('arg')) %}

Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/equality.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_equality(model) %}
{{ adapter.dispatch('test_equality', packages = dbt_utils._get_utils_namespaces())(model, **kwargs) }}
{% endmacro %}

{% macro default__test_equality(model) %}


{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}
Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/expression_is_true.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_expression_is_true(model, condition='true') %}
dataders marked this conversation as resolved.
Show resolved Hide resolved
{{ adapter.dispatch('test_expression_is_true', packages = dbt_utils._get_utils_namespaces())(model, condition='true', **kwargs) }}
{% endmacro %}

{% macro default__test_expression_is_true(model, condition='true') %}

{% set expression = kwargs.get('expression', kwargs.get('arg')) %}

Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/mutually_exclusive_ranges.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed') %}
{{ adapter.dispatch('test_mutually_exclusive_ranges', packages = dbt_utils._get_utils_namespaces())(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed') }}
{% endmacro %}

{% macro default__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed') %}

{% if gaps == 'not_allowed' %}
{% set allow_gaps_operator='=' %}
Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/not_constant.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@

{% macro test_not_constant(model) %}
{{ adapter.dispatch('test_not_constant', packages = dbt_utils._get_utils_namespaces())(model, **kwargs) }}
{% endmacro %}

{% macro default__test_not_constant(model) %}

{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}

Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/recency.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_recency(model, datepart, interval) %}
{{ adapter.dispatch('test_recency', packages = dbt_utils._get_utils_namespaces())(model, datepart, interval, **kwargs) }}
{% endmacro %}

{% macro default__test_recency(model, datepart, interval) %}

{% set column_name = kwargs.get('column_name', kwargs.get('field')) %}

Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/relationships_where.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_relationships_where(model, to, field) %}
{{ adapter.dispatch('test_relationships_where', packages = dbt_utils._get_utils_namespaces())(model, to, field, **kwargs) }}
{% endmacro %}

{% macro default__test_relationships_where(model, to, field) %}

{% set column_name = kwargs.get('column_name', kwargs.get('from')) %}
{% set from_condition = kwargs.get('from_condition', "true") %}
Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/test_not_null_where.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_not_null_where(model) %}
{{ adapter.dispatch('test_not_null_where', packages = dbt_utils._get_utils_namespaces())(model, **kwargs) }}
{% endmacro %}

{% macro default__test_not_null_where(model) %}

{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}
{% set where = kwargs.get('where', kwargs.get('arg')) %}
Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/test_unique_where.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_unique_where(model) %}
{{ adapter.dispatch('test_unique_where', packages = dbt_utils._get_utils_namespaces())(model, **kwargs) }}
{% endmacro %}

{% macro default__test_unique_where(model) %}

{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}
{% set where = kwargs.get('where', kwargs.get('arg')) %}
Expand Down
4 changes: 4 additions & 0 deletions macros/schema_tests/unique_combination_of_columns.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro test_unique_combination_of_columns(model, quote_columns = false) %}
{{ adapter.dispatch('test_unique_combination_of_columns', packages = dbt_utils._get_utils_namespaces())(model, quote_columns = false, **kwargs) }}
{% endmacro %}

{% macro default__test_unique_combination_of_columns(model, quote_columns = false) %}

{%- set columns = kwargs.get('combination_of_columns', kwargs.get('arg')) %}

Expand Down
8 changes: 8 additions & 0 deletions macros/sql/generate_series.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro get_powers_of_two(upper_bound) %}
{{ adapter.dispatch('get_powers_of_two', packages = dbt_utils._get_utils_namespaces())(upper_bound) }}
dataders marked this conversation as resolved.
Show resolved Hide resolved
{% endmacro %}

{% macro default__get_powers_of_two(upper_bound) %}

{% if upper_bound <= 0 %}
{{ exceptions.raise_compiler_error("upper bound must be positive") }}
Expand All @@ -12,6 +16,10 @@


{% macro generate_series(upper_bound) %}
{{ adapter.dispatch('generate_series', packages = dbt_utils._get_utils_namespaces())(upper_bound) }}
{% endmacro %}

{% macro default__generate_series(upper_bound) %}

{% set n = dbt_utils.get_powers_of_two(upper_bound) %}

Expand Down
4 changes: 4 additions & 0 deletions macros/sql/get_column_values.sql
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ Returns:
#}

{% macro get_column_values(table, column, max_records=none, default=none) -%}
{{ adapter.dispatch('get_column_values', packages = dbt_utils._get_utils_namespaces())(table, column, max_records=none, default=none) }}
{% endmacro %}

{% macro default__get_column_values(table, column, max_records=none, default=none) -%}

{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}
{%- if not execute -%}
Expand Down
5 changes: 5 additions & 0 deletions macros/sql/get_query_results_as_dict.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
{% macro get_query_results_as_dict(query) %}
{{ adapter.dispatch('get_query_results_as_dict', packages = dbt_utils._get_utils_namespaces())(query) }}
{% endmacro %}

{% macro default__get_query_results_as_dict(query) %}

{# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #}

{%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%}
Expand Down
4 changes: 4 additions & 0 deletions macros/sql/get_relations_by_pattern.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %}
{{ adapter.dispatch('get_relations_by_pattern', packages = dbt_utils._get_utils_namespaces())(schema_pattern, table_pattern, exclude='', database=target.database) }}
{% endmacro %}

{% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %}

{%- call statement('get_tables', fetch_result=True) %}

Expand Down
4 changes: 4 additions & 0 deletions macros/sql/get_relations_by_prefix.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %}
{{ adapter.dispatch('get_relations_by_prefix', packages = dbt_utils._get_utils_namespaces())(schema, prefix, exclude='', database=target.database) }}
{% endmacro %}

{% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %}

{%- call statement('get_tables', fetch_result=True) %}

Expand Down
6 changes: 5 additions & 1 deletion macros/sql/get_tables_by_prefix_sql.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
{% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %}

{{ adapter.dispatch('get_tables_by_prefix_sql', packages = dbt_utils._get_utils_namespaces())(schema, prefix, exclude='', database=target.database) }}
{% endmacro %}

{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %}

{{ dbt_utils.get_tables_by_pattern_sql(
schema_pattern = schema,
table_pattern = prefix ~ '%',
Expand Down
4 changes: 4 additions & 0 deletions macros/sql/groupby.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{%- macro group_by(n) -%}
{{ adapter.dispatch('group_by', packages = dbt_utils._get_utils_namespaces())(n) }}
{% endmacro %}

{%- macro default__group_by(n) -%}

group by {% for i in range(1, n + 1) -%}
{{ i }}{{ ',' if not loop.last }}
Expand Down
4 changes: 4 additions & 0 deletions macros/sql/nullcheck.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro nullcheck(cols) %}
{{ adapter.dispatch('nullcheck', packages = dbt_utils._get_utils_namespaces())(cols) }}
{% endmacro %}

{% macro default__nullcheck(cols) %}
{%- for col in cols %}

{% if col.is_string() -%}
Expand Down
4 changes: 4 additions & 0 deletions macros/sql/nullcheck_table.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro nullcheck_table(relation) %}
{{ adapter.dispatch('nullcheck_table', packages = dbt_utils._get_utils_namespaces())(relation) }}
{% endmacro %}

{% macro default__nullcheck_table(relation) %}

{%- do dbt_utils._is_relation(relation, 'nullcheck_table') -%}
{%- do dbt_utils._is_ephemeral(relation, 'nullcheck_table') -%}
Expand Down
14 changes: 14 additions & 0 deletions macros/sql/pivot.sql
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,20 @@ Arguments:
else_value=0,
quote_identifiers=True,
distinct=False) %}
{{ adapter.dispatch('pivot', packages = dbt_utils._get_utils_namespaces())(column, values, alias=True, agg='sum', cmp='=', prefix='', suffix='', then_value=1, else_value=0, quote_identifiers=True, distinct=False) }}
{% endmacro %}

{% macro default__pivot(column,
values,
alias=True,
agg='sum',
cmp='=',
prefix='',
suffix='',
then_value=1,
else_value=0,
quote_identifiers=True,
distinct=False) %}
dataders marked this conversation as resolved.
Show resolved Hide resolved
{% for v in values %}
{{ agg }}(
{% if distinct %} distinct {% endif %}
Expand Down
4 changes: 4 additions & 0 deletions macros/sql/safe_add.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{%- macro safe_add() -%}
{{ adapter.dispatch('safe_add', packages = dbt_utils._get_utils_namespaces())() }}
{% endmacro %}

{%- macro default__safe_add() -%}

{% set fields = [] %}

Expand Down
3 changes: 3 additions & 0 deletions macros/sql/star.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
{% macro star(from, relation_alias=False, except=[]) -%}
{{ adapter.dispatch('star', packages = dbt_utils._get_utils_namespaces())(from, relation_alias=False, except=[]) }}
{% endmacro %}

{% macro default__star(from, relation_alias=False, except=[]) -%}
{%- do dbt_utils._is_relation(from, 'star') -%}
{%- do dbt_utils._is_ephemeral(from, 'star') -%}

Expand Down
4 changes: 4 additions & 0 deletions macros/sql/surrogate_key.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{%- macro surrogate_key(field_list) -%}
{{ adapter.dispatch('surrogate_key', packages = dbt_utils._get_utils_namespaces())(field_list) }}
{% endmacro %}

{%- macro default__surrogate_key(field_list) -%}

{%- if varargs|length >= 1 or field_list is string %}

Expand Down
4 changes: 4 additions & 0 deletions macros/sql/union.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%}
{{ adapter.dispatch('union_relations', packages = dbt_utils._get_utils_namespaces())(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') }}
{% endmacro %}

{%- macro default__union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%}

{%- if exclude and include -%}
{{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }}
Expand Down
4 changes: 4 additions & 0 deletions macros/sql/unpivot.sql
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ Arguments:
#}

{% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%}
{{ adapter.dispatch('unpivot', packages = dbt_utils._get_utils_namespaces())(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) }}
{% endmacro %}

{% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%}

{% if table %}
{%- set error_message = '
Expand Down
4 changes: 4 additions & 0 deletions macros/web/get_url_host.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro get_url_host(field) -%}
{{ adapter.dispatch('get_url_host', packages = dbt_utils._get_utils_namespaces())(field) }}
{% endmacro %}

{% macro default__get_url_host(field) -%}

{%- set parsed =
dbt_utils.split_part(
Expand Down
4 changes: 4 additions & 0 deletions macros/web/get_url_parameter.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro get_url_parameter(field, url_parameter) -%}
{{ adapter.dispatch('get_url_parameter', packages = dbt_utils._get_utils_namespaces())(field, url_parameter) }}
{% endmacro %}

{% macro default__get_url_parameter(field, url_parameter) -%}

{%- set formatted_url_parameter = "'" + url_parameter + "='" -%}

Expand Down
4 changes: 4 additions & 0 deletions macros/web/get_url_path.sql
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
{% macro get_url_path(field) -%}
{{ adapter.dispatch('get_url_path', packages = dbt_utils._get_utils_namespaces())(field) }}
{% endmacro %}

{% macro default__get_url_path(field) -%}

{%- set stripped_url =
dbt_utils.replace(
Expand Down