From 7bfa1d96edea928040964202a307d2f50e07b5cd Mon Sep 17 00:00:00 2001 From: georgewoodhead <50772749+georgewoodhead@users.noreply.github.com> Date: Thu, 4 Aug 2022 09:31:03 +0100 Subject: [PATCH] Fix initial dbt compile error (Close #69) --- .../get_incremental_manifest_status.sql | 48 ++++++++++----- .../return_base_new_event_limits.sql | 53 ++++++++++------ macros/utils/return_limits_from_model.sql | 61 ++++++++++++------- 3 files changed, 107 insertions(+), 55 deletions(-) diff --git a/macros/incremental_hooks/get_incremental_manifest_status.sql b/macros/incremental_hooks/get_incremental_manifest_status.sql index e0080161..ee34a0a3 100644 --- a/macros/incremental_hooks/get_incremental_manifest_status.sql +++ b/macros/incremental_hooks/get_incremental_manifest_status.sql @@ -1,31 +1,47 @@ {% macro get_incremental_manifest_status(incremental_manifest_table, models_in_run) -%} + {# In case of not execute just return empty strings to avoid hitting database #} {% if not execute %} - {{ return(['', '', '', '']) }} - {% endif %} - {% set last_success_query %} - select min(last_success) as min_last_success, - max(last_success) as max_last_success, - coalesce(count(*), 0) as models - from {{ incremental_manifest_table }} - where model in ({{ snowplow_utils.print_list(models_in_run) }}) - {% endset %} + {% set target_relation = adapter.get_relation( + database=incremental_manifest_table.database, + schema=incremental_manifest_table.schema, + identifier=incremental_manifest_table.name) %} - {% set results = run_query(last_success_query) %} + {% if target_relation is not none %} - {% if execute %} + {% set last_success_query %} + select min(last_success) as min_last_success, + max(last_success) as max_last_success, + coalesce(count(*), 0) as models + from {{ incremental_manifest_table }} + where model in ({{ snowplow_utils.print_list(models_in_run) }}) + {% endset %} + + {% set results = run_query(last_success_query) %} + + {% if execute %} + + {% set min_last_success = results.columns[0].values()[0] %} + {% set max_last_success = results.columns[1].values()[0] %} + {% set models_matched_from_manifest = results.columns[2].values()[0] %} + {% set has_matched_all_models = true if models_matched_from_manifest == models_in_run|length else false %} + + {{ return([min_last_success, max_last_success, models_matched_from_manifest, has_matched_all_models]) }} + + {% endif %} + + + {% else %} + + {% do exceptions.warn("Snowplow Warning: " ~ incremental_manifest_table ~ " does not exist. This is expected if you are compiling a fresh installation of the dbt-snowplow-* packages.") %} - {% set min_last_success = results.columns[0].values()[0] %} - {% set max_last_success = results.columns[1].values()[0] %} - {% set models_matched_from_manifest = results.columns[2].values()[0] %} - {% set has_matched_all_models = true if models_matched_from_manifest == models_in_run|length else false %} + {{ return(['9999-01-01 00:00:00', '9999-01-01 00:00:00', 0, false]) }} {% endif %} - {{ return([min_last_success, max_last_success, models_matched_from_manifest, has_matched_all_models]) }} {%- endmacro %} diff --git a/macros/incremental_hooks/return_base_new_event_limits.sql b/macros/incremental_hooks/return_base_new_event_limits.sql index e5ae9c11..992c5042 100644 --- a/macros/incremental_hooks/return_base_new_event_limits.sql +++ b/macros/incremental_hooks/return_base_new_event_limits.sql @@ -1,29 +1,48 @@ {% macro return_base_new_event_limits(base_events_this_run) -%} + {# In case of not execute just return empty strings to avoid hitting database #} {% if not execute %} {{ return(['','',''])}} {% endif %} - - {% set limit_query %} - select - lower_limit, - upper_limit, - {{ snowplow_utils.timestamp_add('day', - -var("snowplow__max_session_days", 3), - 'lower_limit') }} as session_start_limit - - from {{ base_events_this_run }} + + {% set target_relation = adapter.get_relation( + database=base_events_this_run.database, + schema=base_events_this_run.schema, + identifier=base_events_this_run.name) %} + + {% if target_relation is not none %} + + {% set limit_query %} + select + lower_limit, + upper_limit, + {{ snowplow_utils.timestamp_add('day', + -var("snowplow__max_session_days", 3), + 'lower_limit') }} as session_start_limit + + from {{ base_events_this_run }} {% endset %} - {% set results = run_query(limit_query) %} - - {% if execute %} + {% set results = run_query(limit_query) %} + + {% if execute %} + + {% set lower_limit = snowplow_utils.cast_to_tstamp(results.columns[0].values()[0]) %} + {% set upper_limit = snowplow_utils.cast_to_tstamp(results.columns[1].values()[0]) %} + {% set session_start_limit = snowplow_utils.cast_to_tstamp(results.columns[2].values()[0]) %} - {% set lower_limit = snowplow_utils.cast_to_tstamp(results.columns[0].values()[0]) %} - {% set upper_limit = snowplow_utils.cast_to_tstamp(results.columns[1].values()[0]) %} - {% set session_start_limit = snowplow_utils.cast_to_tstamp(results.columns[2].values()[0]) %} + {{ return([lower_limit, upper_limit, session_start_limit]) }} - {{ return([lower_limit, upper_limit, session_start_limit]) }} + {% endif %} + + {% else %} + + {% do exceptions.warn("Snowplow Warning: " ~ base_events_this_run ~ " does not exist. This is expected if you are compiling a fresh installation of the dbt-snowplow-* packages.") %} + + {% set dummy_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %} + + {{ return([dummy_limit, dummy_limit, dummy_limit]) }} {% endif %} + {%- endmacro %} diff --git a/macros/utils/return_limits_from_model.sql b/macros/utils/return_limits_from_model.sql index e635b124..244fea87 100644 --- a/macros/utils/return_limits_from_model.sql +++ b/macros/utils/return_limits_from_model.sql @@ -5,33 +5,50 @@ {{ return(['','']) }} {% endif %} - {% set limit_query %} - select - min({{lower_limit_col}}) as lower_limit, - max({{upper_limit_col}}) as upper_limit - from {{ model }} + {% set target_relation = adapter.get_relation( + database=model.database, + schema=model.schema, + identifier=model.name) %} + + {% if target_relation is not none %} + + {% set limit_query %} + select + min({{lower_limit_col}}) as lower_limit, + max({{upper_limit_col}}) as upper_limit + from {{ model }} {% endset %} - {% set results = run_query(limit_query) %} + {% set results = run_query(limit_query) %} - {% if execute %} + {% if execute %} - {# If there is no data within the limits, we should warn them otherwise they may be stuck here forever#} - {%- if results.columns[0].values()[0] is none or results.columns[1].values()[0] is none -%} - {# Currently warnings do not actually do anything other than text in logs, this makes it more visible https://github.com/dbt-labs/dbt-core/issues/6721 #} - {{ snowplow_utils.log_message("Snowplow Warning: *************") }} - {% do exceptions.warn("Snowplow Warning: No data in "~this~" for date range from variables, please modify your run variables to include data if this is not expected.") %} - {{ snowplow_utils.log_message("Snowplow Warning: *************") }} - {# This allows for bigquery to still run the same way the other warehouses do, but also ensures no data is processed #} - {% set lower_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %} - {% set upper_limit = snowplow_utils.cast_to_tstamp('9999-01-02 00:00:00') %} - {%- else -%} - {% set lower_limit = snowplow_utils.cast_to_tstamp(results.columns[0].values()[0]) %} - {% set upper_limit = snowplow_utils.cast_to_tstamp(results.columns[1].values()[0]) %} - {%- endif -%} + {# If there is no data within the limits, we should warn them otherwise they may be stuck here forever#} + {%- if results.columns[0].values()[0] is none or results.columns[1].values()[0] is none -%} + {# Currently warnings do not actually do anything other than text in logs, this makes it more visible https://github.com/dbt-labs/dbt-core/issues/6721 #} + {{ snowplow_utils.log_message("Snowplow Warning: *************") }} + {% do exceptions.warn("Snowplow Warning: No data in "~this~" for date range from variables, please modify your run variables to include data if this is not expected.") %} + {{ snowplow_utils.log_message("Snowplow Warning: *************") }} + {# This allows for bigquery to still run the same way the other warehouses do, but also ensures no data is processed #} + {% set lower_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %} + {% set upper_limit = snowplow_utils.cast_to_tstamp('9999-01-02 00:00:00') %} + {%- else -%} + {% set lower_limit = snowplow_utils.cast_to_tstamp(results.columns[0].values()[0]) %} + {% set upper_limit = snowplow_utils.cast_to_tstamp(results.columns[1].values()[0]) %} + {%- endif -%} + {{ return([lower_limit, upper_limit]) }} - {{ return([lower_limit, upper_limit]) }} + {% endif %} + + {% else %} + + {% do exceptions.warn("Snowplow Warning: " ~ model ~ " does not exist. This is expected if you are compiling a fresh installation of the dbt-snowplow-* packages.") %} + {% set dummy_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %} + + {{ return([dummy_limit, dummy_limit]) }} {% endif %} -{%- endmacro %} + + +{% endmacro %}