Skip to content

Commit

Permalink
Fix initial dbt compile error (Close #69)
Browse files Browse the repository at this point in the history
  • Loading branch information
@georgewoodhead @rlh1994
georgewoodhead authored and rlh1994 committed Feb 20, 2023
1 parent 171ad18 commit 7bfa1d9
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 55 deletions.
48 changes: 32 additions & 16 deletions macros/incremental_hooks/get_incremental_manifest_status.sql
View file
@@ -1,31 +1,47 @@
{% macro get_incremental_manifest_status(incremental_manifest_table, models_in_run) -%}

{# In case of not execute just return empty strings to avoid hitting database #}
{% if not execute %}

{{ return(['', '', '', '']) }}

{% endif %}

{% set last_success_query %}
select min(last_success) as min_last_success,
max(last_success) as max_last_success,
coalesce(count(*), 0) as models
from {{ incremental_manifest_table }}
where model in ({{ snowplow_utils.print_list(models_in_run) }})
{% endset %}
{% set target_relation = adapter.get_relation(
database=incremental_manifest_table.database,
schema=incremental_manifest_table.schema,
identifier=incremental_manifest_table.name) %}

{% set results = run_query(last_success_query) %}
{% if target_relation is not none %}

{% if execute %}
{% set last_success_query %}
select min(last_success) as min_last_success,
max(last_success) as max_last_success,
coalesce(count(*), 0) as models
from {{ incremental_manifest_table }}
where model in ({{ snowplow_utils.print_list(models_in_run) }})
{% endset %}

{% set results = run_query(last_success_query) %}

{% if execute %}

{% set min_last_success = results.columns[0].values()[0] %}
{% set max_last_success = results.columns[1].values()[0] %}
{% set models_matched_from_manifest = results.columns[2].values()[0] %}
{% set has_matched_all_models = true if models_matched_from_manifest == models_in_run|length else false %}

{{ return([min_last_success, max_last_success, models_matched_from_manifest, has_matched_all_models]) }}

{% endif %}


{% else %}

{% do exceptions.warn("Snowplow Warning: " ~ incremental_manifest_table ~ " does not exist. This is expected if you are compiling a fresh installation of the dbt-snowplow-* packages.") %}

{% set min_last_success = results.columns[0].values()[0] %}
{% set max_last_success = results.columns[1].values()[0] %}
{% set models_matched_from_manifest = results.columns[2].values()[0] %}
{% set has_matched_all_models = true if models_matched_from_manifest == models_in_run|length else false %}
{{ return(['9999-01-01 00:00:00', '9999-01-01 00:00:00', 0, false]) }}

{% endif %}

{{ return([min_last_success, max_last_success, models_matched_from_manifest, has_matched_all_models]) }}

{%- endmacro %}

Expand Down
53 changes: 36 additions & 17 deletions macros/incremental_hooks/return_base_new_event_limits.sql
View file
@@ -1,29 +1,48 @@
{% macro return_base_new_event_limits(base_events_this_run) -%}

{# In case of not execute just return empty strings to avoid hitting database #}
{% if not execute %}
{{ return(['','',''])}}
{% endif %}

{% set limit_query %}
select
lower_limit,
upper_limit,
{{ snowplow_utils.timestamp_add('day',
-var("snowplow__max_session_days", 3),
'lower_limit') }} as session_start_limit

from {{ base_events_this_run }}

{% set target_relation = adapter.get_relation(
database=base_events_this_run.database,
schema=base_events_this_run.schema,
identifier=base_events_this_run.name) %}

{% if target_relation is not none %}

{% set limit_query %}
select
lower_limit,
upper_limit,
{{ snowplow_utils.timestamp_add('day',
-var("snowplow__max_session_days", 3),
'lower_limit') }} as session_start_limit

from {{ base_events_this_run }}
{% endset %}

{% set results = run_query(limit_query) %}

{% if execute %}
{% set results = run_query(limit_query) %}

{% if execute %}

{% set lower_limit = snowplow_utils.cast_to_tstamp(results.columns[0].values()[0]) %}
{% set upper_limit = snowplow_utils.cast_to_tstamp(results.columns[1].values()[0]) %}
{% set session_start_limit = snowplow_utils.cast_to_tstamp(results.columns[2].values()[0]) %}

{% set lower_limit = snowplow_utils.cast_to_tstamp(results.columns[0].values()[0]) %}
{% set upper_limit = snowplow_utils.cast_to_tstamp(results.columns[1].values()[0]) %}
{% set session_start_limit = snowplow_utils.cast_to_tstamp(results.columns[2].values()[0]) %}
{{ return([lower_limit, upper_limit, session_start_limit]) }}

{{ return([lower_limit, upper_limit, session_start_limit]) }}
{% endif %}

{% else %}

{% do exceptions.warn("Snowplow Warning: " ~ base_events_this_run ~ " does not exist. This is expected if you are compiling a fresh installation of the dbt-snowplow-* packages.") %}

{% set dummy_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %}

{{ return([dummy_limit, dummy_limit, dummy_limit]) }}

{% endif %}

{%- endmacro %}
61 changes: 39 additions & 22 deletions macros/utils/return_limits_from_model.sql
View file
Expand Up @@ -5,33 +5,50 @@
{{ return(['','']) }}
{% endif %}

{% set limit_query %}
select
min({{lower_limit_col}}) as lower_limit,
max({{upper_limit_col}}) as upper_limit
from {{ model }}
{% set target_relation = adapter.get_relation(
database=model.database,
schema=model.schema,
identifier=model.name) %}

{% if target_relation is not none %}

{% set limit_query %}
select
min({{lower_limit_col}}) as lower_limit,
max({{upper_limit_col}}) as upper_limit
from {{ model }}
{% endset %}

{% set results = run_query(limit_query) %}
{% set results = run_query(limit_query) %}

{% if execute %}
{% if execute %}

{# If there is no data within the limits, we should warn them otherwise they may be stuck here forever#}
{%- if results.columns[0].values()[0] is none or results.columns[1].values()[0] is none -%}
{# Currently warnings do not actually do anything other than text in logs, this makes it more visible https://github.com/dbt-labs/dbt-core/issues/6721 #}
{{ snowplow_utils.log_message("Snowplow Warning: *************") }}
{% do exceptions.warn("Snowplow Warning: No data in "~this~" for date range from variables, please modify your run variables to include data if this is not expected.") %}
{{ snowplow_utils.log_message("Snowplow Warning: *************") }}
{# This allows for bigquery to still run the same way the other warehouses do, but also ensures no data is processed #}
{% set lower_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %}
{% set upper_limit = snowplow_utils.cast_to_tstamp('9999-01-02 00:00:00') %}
{%- else -%}
{% set lower_limit = snowplow_utils.cast_to_tstamp(results.columns[0].values()[0]) %}
{% set upper_limit = snowplow_utils.cast_to_tstamp(results.columns[1].values()[0]) %}
{%- endif -%}
{# If there is no data within the limits, we should warn them otherwise they may be stuck here forever#}
{%- if results.columns[0].values()[0] is none or results.columns[1].values()[0] is none -%}
{# Currently warnings do not actually do anything other than text in logs, this makes it more visible https://github.com/dbt-labs/dbt-core/issues/6721 #}
{{ snowplow_utils.log_message("Snowplow Warning: *************") }}
{% do exceptions.warn("Snowplow Warning: No data in "~this~" for date range from variables, please modify your run variables to include data if this is not expected.") %}
{{ snowplow_utils.log_message("Snowplow Warning: *************") }}
{# This allows for bigquery to still run the same way the other warehouses do, but also ensures no data is processed #}
{% set lower_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %}
{% set upper_limit = snowplow_utils.cast_to_tstamp('9999-01-02 00:00:00') %}
{%- else -%}
{% set lower_limit = snowplow_utils.cast_to_tstamp(results.columns[0].values()[0]) %}
{% set upper_limit = snowplow_utils.cast_to_tstamp(results.columns[1].values()[0]) %}
{%- endif -%}

{{ return([lower_limit, upper_limit]) }}

{{ return([lower_limit, upper_limit]) }}
{% endif %}

{% else %}

{% do exceptions.warn("Snowplow Warning: " ~ model ~ " does not exist. This is expected if you are compiling a fresh installation of the dbt-snowplow-* packages.") %}
{% set dummy_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %}

{{ return([dummy_limit, dummy_limit]) }}

{% endif %}
{%- endmacro %}


{% endmacro %}

0 comments on commit 7bfa1d9

Please sign in to comment.