feat(): add ad_reporting passthrough metrics variable

CHANGELOG.md

@@ -1,3 +1,11 @@
+# dbt_ad_reporting v1.3.2
+
+## Feature Enhancement
+- Added `ad_reporting__<report>_passthrough_metrics` variables to easily add common metrics across all platforms into the `ad_reporting` models.
+
+## Contributors
+- [@aleix-cd](https://github.com/aleix-cd)
+
 # dbt_ad_reporting v1.3.1
 
 ## Updates

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'ad_reporting'
-version: '1.3.1'
+version: '1.3.2'
 
 config-version: 2
 
@@ -11,4 +11,4 @@ models:
     +schema: ad_reporting 
     intermediate:
       +materialized: ephemeral
-      +schema: int_ad_reporting 
+      +schema: int_ad_reporting

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'ad_reporting_integration_tests'
-version: '1.3.1'
+version: '1.3.2'
 profile: 'integration_tests'
 config-version: 2
 

macros/get_query.sql

@@ -28,41 +28,88 @@
 {#- For campaign level reports and lower, add campaign_fields -#}
 {%- if report_type in ['campaign', 'ad_group', 'ad', 'url', 'keyword', 'search'] -%}
     {%- for campaign_field in campaign_fields -%}
+        {#- When campaign_passthrough_metrics are defined, add them too but only to the ad_group report_type -#}
+        {%- if report_type == 'campaign' and var('ad_reporting__campaign_passthrough_metrics', []) -%}
+            {% set campaign_passthrough_metric_array_of_dicts = var('ad_reporting__campaign_passthrough_metrics') %}
+                {%- for campaign_passthrough_metric_dict in campaign_passthrough_metric_array_of_dicts -%}
+                    {%- for campaign_passthrough_metric_value in campaign_passthrough_metric_dict.values() -%}
+                        {%- do final_fields_superset.update({campaign_passthrough_metric_value: campaign_passthrough_metric_value}) -%}
+                    {%- endfor -%}
+                {%- endfor -%}
+        {%- endif -%}
         {%- do final_fields_superset.update({campaign_field: campaign_field}) -%}
     {%- endfor -%}
 {%- endif -%}
 
 {#- For ad_group level reports, equivalent and lower, add ad_group_fields -#}
 {%- if report_type in ['ad_group', 'ad', 'url', 'keyword', 'search'] -%}
     {%- for ad_group_field in ad_group_fields -%}
+        {#- When ad_group_passthrough_metrics are defined, add them too but only to the ad_group report_type -#}
+        {%- if report_type == 'ad_group' and var('ad_reporting__ad_group_passthrough_metrics', []) -%}
+            {% set ad_group_passthrough_metric_array_of_dicts = var('ad_reporting__ad_group_passthrough_metrics') %}
+                {%- for ad_group_passthrough_metric_dict in ad_group_passthrough_metric_array_of_dicts -%}
+                    {%- for ad_group_passthrough_metric_value in ad_group_passthrough_metric_dict.values() -%}
+                        {%- do final_fields_superset.update({ad_group_passthrough_metric_value: ad_group_passthrough_metric_value}) -%}
+                    {%- endfor -%}
+                {%- endfor -%}
+        {%- endif -%}
         {%- do final_fields_superset.update({ad_group_field: ad_group_field}) -%}
     {%- endfor -%}
 {%- endif -%}
 
-{#- For ad level reports, add ad_fields -#}
-{%- if report_type == 'ad' -%}
-    {%- for ad_field in ad_fields -%}
+{#- For ad and url level reports, add ad_fields and ad_passthrough_metrics (if any) -#}
+{%- if report_type in ['ad', 'url'] -%}
+    {%- if var('ad_reporting__ad_passthrough_metrics', []) -%}
+        {%- set ad_passthrough_metrics_values = [] -%}
+        {%- set ad_passthrough_metrics_array_of_dicts = var('ad_reporting__ad_passthrough_metrics') -%}
+            {%- for ad_passthrough_metrics_dict in ad_passthrough_metrics_array_of_dicts -%}
+                {%- for _, value in ad_passthrough_metrics_dict.items() -%}
+                    {%- do ad_passthrough_metrics_values.append(value) -%}
+                {%- endfor -%}
+            {%- endfor -%}
+        {%- set combined_ad_fields = ad_fields + ad_passthrough_metrics_values -%}
+    {%- else -%}
+        {%- set combined_ad_fields = ad_fields -%}
+    {%- endif -%}
+    {%- for ad_field in combined_ad_fields -%}
         {%- do final_fields_superset.update({ad_field: ad_field})-%}
     {%- endfor -%}
 {%- endif -%}
 
-{#- For url level reports, add url_fields -#}
-{%- if report_type == 'url' -%}
-    {%- for url_field in url_fields -%}
-        {%- do final_fields_superset.update({url_field: url_field})-%}
-    {%- endfor -%}
-{%- endif -%}
-
-{#- For keyword level reports, add keyword_fields -#}
+{#- For keyword level reports, add keyword_fields and keyword_passthrough_metrics (if any) -#}
 {%- if report_type == 'keyword' -%}
-    {%- for keyword_field in keyword_fields -%}
+    {%- if var('ad_reporting__keyword_passthrough_metrics', []) -%}
+        {%- set keyword_passthrough_metrics_values = [] -%}
+        {%- set keyword_passthrough_metrics_array_of_dicts = var('ad_reporting__keyword_passthrough_metrics') -%}
+            {%- for keyword_passthrough_metrics_dict in keyword_passthrough_metrics_array_of_dicts -%}
+                {%- for _, value in keyword_passthrough_metrics_dict.items() -%}
+                    {%- do keyword_passthrough_metrics_values.append(value) -%}
+                {%- endfor -%}
+            {%- endfor -%}
+        {%- set combined_keyword_fields = keyword_fields + keyword_passthrough_metrics_values -%}
+    {%- else -%}
+        {%- set combined_keyword_fields = keyword_fields -%}
+    {%- endif -%}
+    {%- for keyword_field in combined_keyword_fields -%}
         {%- do final_fields_superset.update({keyword_field: keyword_field})-%}
     {%- endfor -%}
 {%- endif -%}
 
-{#- For search level reports, add search_fields -#}
+{#- For search level reports, add search_fields and search_passthrough_metrics (if any) -#}
 {%- if report_type == 'search' -%}
-    {%- for search_field in search_fields -%}
+    {%- if var('ad_reporting__search_passthrough_metrics') -%}
+        {%- set search_passthrough_metrics_values = [] -%}
+        {%- set search_passthrough_metrics_array_of_dicts = var('ad_reporting__search_passthrough_metrics') -%}
+            {%- for search_passthrough_metrics_dict in search_passthrough_metrics_array_of_dicts -%}
+                {%- for _, value in search_passthrough_metrics_dict.items() -%}
+                    {%- do search_passthrough_metrics_values.append(value) -%}
+                {%- endfor -%}
+            {%- endfor -%}
+        {%- set combined_search_fields = search_fields + search_passthrough_metrics_values -%}
+    {%- else -%}
+        {%- set combined_search_fields = search_fields -%}
+    {%- endif -%}
+    {%- for search_field in combined_search_fields -%}
         {%- do final_fields_superset.update({search_field: search_field})-%}
     {%- endfor -%}
 {%- endif -%}
@@ -78,14 +125,18 @@ select
     cast( '{{ platform }}' as {{ dbt.type_string() }}) as platform,
 
     {% for field in final_fields_superset.keys()|sort() -%}
-    {% if field in ['clicks', 'impressions'] -%}
+    {% if field in consistent_fields and field != 'spend' -%}
     cast({{ final_fields_superset[field] }} as {{ dbt.type_int() }}) as {{ field }}
 
     {% elif field == 'spend' -%}
     cast({{ final_fields_superset[field] }} as {{ dbt.type_float() }}) as {{ field }}
 
     {% elif '_id' in field or '_name' in field or 'url' in field or 'utm' in field or field in ['keyword_match_type', 'keyword_text', 'search_match_type', 'search_query'] -%}
-    cast({{ final_fields_superset[field] }} as {{ dbt.type_string() }}) as {{ field }} 
+    cast({{ final_fields_superset[field] }} as {{ dbt.type_string() }}) as {{ field }}
+
+    {# This is the case for the rest of fields (passthrough_metrics) #}
+    {% else %}
+    cast({{ final_fields_superset[field] }} as {{ dbt.type_float() }}) as {{ field }}
     {% endif -%}
     {%- if not loop.last -%},{%- endif -%}
     {%- endfor %}

models/ad_reporting__account_report.sql

@@ -16,7 +16,10 @@ aggregated as (
         account_name,
         sum(clicks) as clicks,
         sum(impressions) as impressions,
-        sum(spend) as spend
+        sum(spend) as spend 
+
+        {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='ad_reporting__account_passthrough_metrics', transform = 'sum') }}
+
     from base
     {{ dbt_utils.group_by(4) }}
 )

models/ad_reporting__ad_group_report.sql

@@ -20,7 +20,10 @@ aggregated as (
         ad_group_name,
         sum(clicks) as clicks,
         sum(impressions) as impressions,
-        sum(spend) as spend
+        sum(spend) as spend 
+
+        {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='ad_reporting__ad_group_passthrough_metrics', transform = 'sum') }}
+
     from base
     {{ dbt_utils.group_by(8) }}
 )

models/ad_reporting__ad_report.sql

@@ -22,7 +22,10 @@ aggregated as (
         ad_name,
         sum(clicks) as clicks,
         sum(impressions) as impressions,
-        sum(spend) as spend
+        sum(spend) as spend 
+
+        {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='ad_reporting__ad_passthrough_metrics', transform = 'sum') }}
+
     from base
     {{ dbt_utils.group_by(10) }}
 )

models/ad_reporting__campaign_report.sql

@@ -18,7 +18,10 @@ aggregated as (
         campaign_name,
         sum(clicks) as clicks,
         sum(impressions) as impressions,
-        sum(spend) as spend
+        sum(spend) as spend 
+
+        {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='ad_reporting__campaign_passthrough_metrics', transform = 'sum') }}
+
     from base
     {{ dbt_utils.group_by(6) }}
 )

models/ad_reporting__keyword_report.sql

@@ -29,7 +29,10 @@ aggregated as (
         keyword_match_type,
         sum(clicks) as clicks,
         sum(impressions) as impressions,
-        sum(spend) as spend
+        sum(spend) as spend 
+
+        {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='ad_reporting__keyword_passthrough_metrics', transform = 'sum') }}
+
     from base
     {{ dbt_utils.group_by(11) }}
 )

models/ad_reporting__search_report.sql

@@ -30,7 +30,10 @@ aggregated as (
         search_match_type,
         sum(clicks) as clicks,
         sum(impressions) as impressions,
-        sum(spend) as spend
+        sum(spend) as spend 
+
+        {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='ad_reporting__search_passthrough_metrics', transform = 'sum') }}
+
     from base
     {{ dbt_utils.group_by(12) }}
 )

models/ad_reporting__url_report.sql

@@ -28,7 +28,10 @@ aggregated as (
         utm_term,
         sum(clicks) as clicks,
         sum(impressions) as impressions,
-        sum(spend) as spend
+        sum(spend) as spend 
+
+        {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='ad_reporting__url_passthrough_metrics', transform = 'sum') }}
+
     from base
     {{ dbt_utils.group_by(16) }}
 )