Adapt get_incremental_manifest_status macro to be compatible with dbt compile

[emielver]

Describe the feature

When you run dbt compile on a fresh installation of the dbt-snowplow-* packages you get a compilation error since the get_incremental_manifest_status macro tries to reference tables that do not exist. A user has kindly offered the following macro (for snowflake) that should solve this problem. Perhaps we can explore this macro and modify it to be compatible with all databases such that the dbt compile command runs without errors on initial use.

{% macro get_incremental_manifest_status(incremental_manifest_table, models_in_run) -%}
  {% if not execute %}
    {{ return(['', '', '', '']) }}
  {% endif %}
  {% set table_exists_query %}
    select exists(select 1 as x from information_schema.tables t where '"'+current_database()+'"'+'.'+'"'+table_schema+'"'+'.'+'"'+t.table_name+'"' = '{{ incremental_manifest_table }}') as has_tbl
  {% endset %}
  {% set table_exists_query_result = run_query(table_exists_query) %}
  {% do log(table_exists_query_result.columns[0]) %}
  {{ log('log0') }}
  {% if table_exists_query_result.columns[0].values()[0] == True %}
    {{ log('log01') }}
    {% set last_success_query %}
      select min(last_success) as min_last_success,
            max(last_success) as max_last_success,
            coalesce(count(*), 0) as models
      from {{ incremental_manifest_table }}
      where model in ({{ snowplow_utils.print_list(models_in_run) }})
    {% endset %}
    {{ log('log02') }}
    {% set results = run_query(last_success_query) %}
    {{ log('log1') }}
    {% if execute %}
      {{ log('log2') }}
      {% set min_last_success = results.columns[0].values()[0] %}
      {% set max_last_success = results.columns[1].values()[0] %}
      {% set models_matched_from_manifest = results.columns[2].values()[0] %}
      {% set has_matched_all_models = true if models_matched_from_manifest == models_in_run|length else false %}
    {% endif %}
    {{ return([min_last_success, max_last_success, models_matched_from_manifest, has_matched_all_models]) }}
  {% else %}
    {{ log('Table does not exists.') }}
    {{ return(['', '', '', '']) }}
  {% endif %}
{%- endmacro %}
{# Prints the run limits for the run to the console #}
{% macro print_run_limits(run_limits_relation) -%}
  {% set run_limits_query %}
    select lower_limit, upper_limit from {{ run_limits_relation }}
  {% endset %}
  {# Derive limits from manifest instead of selecting from limits table since run_query executes during 2nd parse the limits table is yet to be updated. #}
  {% set results = run_query(run_limits_query) %}
  {% if execute %}
    {% set lower_limit = snowplow_utils.tstamp_to_str(results.columns[0].values()[0]) %}
    {% set upper_limit = snowplow_utils.tstamp_to_str(results.columns[1].values()[0]) %}
    {% set run_limits_message = "Snowplow: Processing data between " + lower_limit + " and " + upper_limit %}
    {% do snowplow_utils.log_message(run_limits_message) %}
  {% endif %}
{%- endmacro %}

[georgewoodhead]

We have begun looking into this issue under branch feature/initial_dbt_compile_fix. Whilst this does resolve the error in the get_incremental_manifest_status macro when running dbt compile on a fresh installation, downstream models will still return a SQL compilation error from non-existent derived, scratch or snowplow_manifest tables.

An alternative solution if you needed to run dbt compile before dbt run on a fresh installation would be to create the empty tables that are referenced by other models beforehand.