Bug/account daily overview query resources exceeded

CHANGELOG.md

@@ -1,3 +1,10 @@
+
+# dbt_recurly v0.2.1
+# Features  
+- Refactored the `recurly__account_daily_overview` to prevent query timeouts, brought up in [#8](https://github.com/fivetran/dbt_recurly/issues/8).
+- Created intermediate models previously present in `recurly__account_daily_overview` to optimize runtime for advanced loop functions, `int_recurly__account_partitions` and `int_recurly__account_running_totals`. 
+- Changed intermediate folder materializations from ephemeral to table because of complexity of calculations. 
+
 # dbt_recurly v0.2.0
 ## 🚨 Breaking Changes 🚨:
 [PR #7](https://github.com/fivetran/dbt_recurly/pull/7) includes the following breaking changes:
@@ -42,6 +49,7 @@
 
 ## Contributors
 - [@suelai](https://github.com/suelai) ([#10](https://github.com/fivetran/dbt_recurly/pull/10))
+
 # dbt_recurly v0.1.0
 
 ## Initial Release

dbt_project.yml

@@ -1,14 +1,15 @@
 config-version: 2
 name: 'recurly'
-version: '0.2.0'
+
+version: '0.2.1'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:
   recurly:
     +schema: recurly
     +materialized: table
     intermediate:
-      +materialized: ephemeral
+      +materialized: table
 vars:
   recurly:
     account_balance_history: "{{ ref('stg_recurly__account_balance_history') }}"

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'recurly_integration_tests'
-version: '0.2.0'
+version: '0.1.1'
 profile: 'integration_tests'
 config-version: 2
 

models/intermediate/int_recurly__account_cumulatives.sql

@@ -13,31 +13,31 @@ balance_transaction_joined as (
 account_current_month as (
 
         select account_id,
-                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp_backcompat()) }}
+                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp()) }}
                         then daily_transactions
                         else 0 
                         end) as transactions_this_month,
-                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp_backcompat()) }}
+                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp()) }}
                         then daily_invoices
                         else 0 
                         end) as invoices_this_month,
-                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp_backcompat()) }}
+                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp()) }}
                         then daily_balance
                         else 0 
                         end) as balance_this_month,
-                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp_backcompat()) }}
+                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp()) }}
                         then daily_charges
                         else 0 
                         end) as charges_this_month,
-                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp_backcompat()) }}
+                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp()) }}
                         then daily_credits
                         else 0 
                         end) as credits_this_month,
-                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp_backcompat()) }}
+                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp()) }}
                         then daily_discounts
                         else 0 
                         end) as discounts_this_month,
-                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp_backcompat()) }}
+                sum(case when {{ dbt.date_trunc('month', 'date_day') }} = {{ dbt.date_trunc('month', dbt.current_timestamp()) }}
                         then daily_credits
                         else 0 
                         end) as taxes_this_month

models/intermediate/int_recurly__account_partitions.sql

@@ -0,0 +1,25 @@
+{% set fields = ['rolling_account_balance','rolling_invoices','rolling_transactions','rolling_charge_balance','rolling_credit_balance','rolling_discount_balance','rolling_tax_balance','rolling_charges','rolling_credits'] %}
+
+with account_rolling_totals as (
+
+    select * 
+    from {{ ref('int_recurly__account_rolling_totals') }}
+),
+
+
+final as (
+
+    select
+        *,
+        {% for f in fields %}
+        sum(case when {{ f }} is null  
+            then 0  
+            else 1  
+                end) over (order by account_id, date_day rows unbounded preceding) as {{ f }}_partition
+        {%- if not loop.last -%},{%- endif -%}
+        {% endfor %}                  
+    from account_rolling_totals
+)
+
+select * 
+from final

models/intermediate/int_recurly__account_running_totals.sql

@@ -0,0 +1,61 @@
+{% set fields = ['rolling_account_balance','rolling_invoices','rolling_transactions','rolling_charge_balance','rolling_credit_balance','rolling_discount_balance','rolling_tax_balance','rolling_charges','rolling_credits'] %}
+
+with account_partitions as (
+
+    select * 
+    from {{ ref('int_recurly__account_partitions') }}
+),
+
+account_overview as (
+
+    select * 
+    from {{ ref('recurly__account_overview') }}
+),
+
+final as (
+
+    select
+        account_partitions.account_id,
+        account_overview.account_created_at,
+        account_overview.account_city,
+        account_overview.account_company,
+        account_overview.account_country,
+        account_overview.account_code,
+        account_overview.account_email,
+        account_overview.account_first_name,
+        account_overview.account_last_name,
+        account_overview.account_is_tax_exempt,
+        account_overview.account_postal_code,
+        account_overview.account_region,
+        account_overview.account_state,
+        account_overview.account_username
+
+        {{ fivetran_utils.persist_pass_through_columns('recurly_account_pass_through_columns', identifier='account_overview') }},      
+        {{ dbt_utils.generate_surrogate_key(['account_partitions.account_id','date_day']) }} as account_daily_id,
+
+        date_day,        
+        date_week, 
+        date_month, 
+        date_year,  
+        date_index, 
+        coalesce(daily_transactions,0) as daily_transaction_count,
+        coalesce(daily_balance,0) as daily_net_change,
+        coalesce(daily_invoices,0) as daily_invoice_count,
+        coalesce(daily_charges,0) as daily_charges,
+        coalesce(daily_credits,0) as daily_credits,
+        coalesce(daily_discounts,0) as daily_discounts,
+        coalesce(daily_taxes,0) as daily_taxes,
+        coalesce(daily_charge_count,0) as daily_charge_count,
+        coalesce(daily_credit_count,0) as daily_credit_count,
+        {% for f in fields %}
+        coalesce({{ f }},   
+            first_value({{ f }}) over (partition by {{ f }}_partition order by date_day rows unbounded preceding)) as {{ f }}
+        {%- if not loop.last -%},{%- endif -%}
+        {% endfor %}
+    from account_partitions
+    left join account_overview
+        on account_partitions.account_id = account_overview.account_id
+)    
+
+select *
+from final

models/recurly.yml

@@ -3,10 +3,16 @@ version: 2
 models:
   - name: int_recurly__account_cumulatives
     description: Each record is an account and its aggregates in terms of balance and record history.
+
+  - name: int_recurly__account_partitions
+    description: Each record is a group of partitioned account totals updating null values with zeroes to eventually calculate running totals downstream.
 
   - name: int_recurly__account_rolling_totals
     description: Each record is a historical look at an account's balance and its changes over time. 
 
+  - name: int_recurly__account_running_totals
+    description: Each record is a historical look at an account's balance, changes over time, and daily running updates to key metrics 
+
   - name: int_recurly__transactions_date_spine
     description: This date spine allows us to construct a full date calendar for account rolling totals.
 

models/recurly__account_daily_overview.sql

@@ -1,75 +1,8 @@
-{% set fields = ['rolling_account_balance','rolling_invoices','rolling_transactions','rolling_charge_balance','rolling_credit_balance','rolling_discount_balance','rolling_tax_balance','rolling_charges','rolling_credits'] %}
-
-with account_rolling_totals as (
+ with final as (
 
     select * 
-    from {{ ref('int_recurly__account_rolling_totals') }}
-),
-
-account_overview as (
-
-    select * 
-    from {{ ref('recurly__account_overview') }}
-),
-
-account_partitions as (
-
-    select
-        *,
-        {% for f in fields %}
-        sum(case when {{ f }} is null  
-            then 0  
-            else 1  
-                end) over (order by account_id, date_day rows unbounded preceding) as {{ f }}_partition
-        {%- if not loop.last -%},{%- endif -%}
-        {% endfor %}                  
-    from account_rolling_totals
-),
-
-final as (
-
-    select
-        account_partitions.account_id,
-        account_overview.account_created_at,
-        account_overview.account_city,
-        account_overview.account_company,
-        account_overview.account_country,
-        account_overview.account_code,
-        account_overview.account_email,
-        account_overview.account_first_name,
-        account_overview.account_last_name,
-        account_overview.account_is_tax_exempt,
-        account_overview.account_postal_code,
-        account_overview.account_region,
-        account_overview.account_state,
-        account_overview.account_username 
-
-        {{ fivetran_utils.persist_pass_through_columns('recurly_account_pass_through_columns', identifier='account_overview') }},
-
-        date_day,        
-        {{ dbt_utils.generate_surrogate_key(['account_partitions.account_id','date_day']) }} as account_daily_id,
-        date_week, 
-        date_month, 
-        date_year,  
-        date_index, 
-        coalesce(daily_transactions,0) as daily_transaction_count,
-        coalesce(daily_balance,0) as daily_net_change,
-        coalesce(daily_invoices,0) as daily_invoice_count,
-        coalesce(daily_charges,0) as daily_charges,
-        coalesce(daily_credits,0) as daily_credits,
-        coalesce(daily_discounts,0) as daily_discounts,
-        coalesce(daily_taxes,0) as daily_taxes,
-        coalesce(daily_charge_count,0) as daily_charge_count,
-        coalesce(daily_credit_count,0) as daily_credit_count,
-        {% for f in fields %}
-        coalesce({{ f }},   
-            first_value({{ f }}) over (partition by {{ f }}_partition order by date_day rows unbounded preceding)) as {{ f }}
-        {%- if not loop.last -%},{%- endif -%}
-        {% endfor %}
-    from account_partitions
-    left join account_overview
-        on account_partitions.account_id = account_overview.account_id
-)    
+    from {{ ref('int_recurly__account_running_totals') }}
+) 
 
 select *
 from final

models/recurly__account_overview.sql

@@ -23,7 +23,7 @@ account_next_invoice as (
         account_id, 
         min(invoice_due_at) as next_invoice_due_at
     from balance_transaction_joined
-    where invoice_due_at > {{ dbt.date_trunc('day', dbt.current_timestamp_backcompat()) }}
+    where invoice_due_at > {{ dbt.date_trunc('day', dbt.current_timestamp()) }}
     group by 1  
 ),
 

packages.yml

@@ -1,3 +1,9 @@
 packages:
-  - package: fivetran/recurly_source
-    version: [">=0.2.0", "<0.3.0"]
+#   - package: fivetran/recurly_source
+#     version: [">=0.2.0", "<0.3.0"]
+
+# IMPORTANT: MAKE SURE TO REVERT BACK BEFORE MERGING
+
+  - git: https://github.com/fivetran/dbt_recurly_source.git
+    revision: MagicBot/dbt-utils-cross-db-migration
+    warn-unpinned: false