Apply v1.3 API updates + remove JSON fields

.gitignore

@@ -3,3 +3,4 @@ target/
 dbt_modules/
 logs/
 dbt_packages/
+env/

CHANGELOG.md

@@ -1,9 +1,15 @@
-# dbt_tiktok_ads v0.UPDATE.UPDATE
+# dbt_tiktok_ads v0.4.0
+[PR #11](https://github.com/fivetran/dbt_tiktok_ads/pull/11) includes the following changes:
 
- ## Under the Hood:
+## 🚨 Breaking Changes 🚨:
+- In the [July 2023 connector update for TikTok Ads](https://fivetran.com/docs/applications/tiktok-ads/changelog), we upgraded the connector to the v1.3 API. As a result the dependent fields and field names from the downstream staging models have changed.
+- None of the renamed fields were brought into the `dbt_tiktok_ads` package other than `age ` to `age_groups`, but it was subsequently removed due to an additional issue. We did need to update seed files with the renamed field names.
+- Removed JSON array fields (`action_categories`, `age_groups`, `interest_category` and `language`) from our models that were causing issues with BigQuery, which currently does not support JSON arrays in group by functions
 
+ ## Under the Hood:
 - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
 - Updated the pull request [templates](/.github).
+
 # dbt_tiktok_ads v0.3.0
 
 ## 🚨 Breaking Changes 🚨:

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'tiktok_ads'
-version: '0.3.0'
+version: '0.4.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/.gitignore

@@ -3,4 +3,5 @@ target/
 dbt_modules/
 logs/
 .DS_Store
-dbt_packages/
+dbt_packages/
+env/

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'tiktok_ads_integration_tests'
-version: '0.3.0'
+version: '0.4.0'
 profile: 'integration_tests'
 config-version: 2
 

integration_tests/seeds/tiktok_ad_history_data.csv

@@ -1,4 +1,4 @@
-AD_ID,UPDATED_AT,ADVERTISER_ID,ADGROUP_ID,CAMPAIGN_ID,CREATE_TIME,AD_NAME,CALL_TO_ACTION,STATUS,OPT_STATUS,AD_TEXT,VIDEO_ID,APP_NAME,OPEN_URL,LANDING_PAGE_URL,DISPLAY_NAME,PROFILE_IMAGE,IMPRESSION_TRACKING_URL,CLICK_TRACKING_URL,PLAYABLE_URL,IS_ACO,IS_CREATIVE_AUTHORIZED,IS_NEW_STRUCTURE,IMAGE_IDS,_FIVETRAN_SYNCED
+AD_ID,UPDATED_AT,ADVERTISER_ID,ADGROUP_ID,CAMPAIGN_ID,CREATE_TIME,AD_NAME,CALL_TO_ACTION,SECONDARY_STATUS,OPERATION_STATUS,AD_TEXT,VIDEO_ID,APP_NAME,DEEPLINK,LANDING_PAGE_URL,DISPLAY_NAME,PROFILE_IMAGE_URL,IMPRESSION_TRACKING_URL,CLICK_TRACKING_URL,PLAYABLE_URL,IS_ACO,CREATIVE_AUTHORIZED,IS_NEW_STRUCTURE,IMAGE_IDS,_FIVETRAN_SYNCED
 1,2021-09-15 04:26:28.000,1,1,1,2021-08-27 14:13:57.000,ad name one,SIGN_UP,AD_STATUS_DELIVERY_OK,DISABLE,subscribe,v10033g50000c4kf54jc77ucvou6fv0g,,,https://www.stuffthatworks.health/,health,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:16.797
 2,2021-09-15 03:19:25.000,2,2,2,2021-09-13 18:54:11.000,ad name two,SIGN_UP,AD_STATUS_DELIVERY_OK,ENABLE,join us,v10033g50000c4vnvmbc77u58bhlklvg,,,https://swaggystocks.com/dashboard/home,stocks,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:16.641
 3,2021-09-15 03:54:51.000,3,3,3,2021-07-26 16:28:39.000,ad name three,INTERESTED,AD_STATUS_DELIVERY_OK,ENABLE,like and share,v10033g50000c3qbo6jc77uaacd9hvv0,,,https://finviz.com/screener.ashx,more stocks,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:18.918

integration_tests/seeds/tiktok_adgroup_history_data.csv

@@ -1,4 +1,4 @@
-ADGROUP_ID,UPDATED_AT,ADVERTISER_ID,CAMPAIGN_ID,CREATE_TIME,ADGROUP_NAME,PLACEMENT_TYPE,PROFILE_IMAGE,LANDING_PAGE_URL,DISPLAY_NAME,APP_TYPE,APP_DOWNLOAD_URL,APP_NAME,EXTERNAL_ACTION,DEEP_EXTERNAL_ACTION,CREATIVE_MATERIAL_MODE,AUDIENCE_TYPE,GENDER,ANDROID_OSV,IOS_OSV,BUDGET_MODE,SCHEDULE_TYPE,DAYPARTING,OPTIMIZE_GOAL,CPV_VIDEO_DURATION,PACING,BILLING_EVENT,BID_TYPE,DEEP_BID_TYPE,IMPRESSION_TRACKING_URL,CLICK_TRACKING_URL,STATUS,OPT_STATUS,STATISTIC_TYPE,VIDEO_DOWNLOAD,OPEN_URL,OPEN_URL_TYPE,FALLBACK_TYPE,BUDGET,BID,CONVERSION_BID,DEEP_CPABID,SCHEDULE_START_TIME,SCHEDULE_END_TIME,APP_ID,PIXEL_ID,ENABLE_INVENTORY_FILTER,IS_HFSS,IS_NEW_STRUCTURE,CATEGORY,IS_COMMENT_DISABLE,SKIP_LEARNING_PHASE,FREQUENCY,FREQUENCY_SCHEDULE,ACTION_DAYS,AUDIENCE,EXCLUDED_AUDIENCE,LOCATION,INTEREST_CATEGORY_V_2,PANGLE_BLOCK_APP_LIST_ID,ACTION_CATEGORIES,PLACEMENT,KEYWORDS,AGE,LANGUAGES,OPERATION_SYSTEM,CONNECTION_TYPE,CARRIERS,VIDEO_ACTIONS,PACKAGE,_FIVETRAN_SYNCED
+ADGROUP_ID,UPDATED_AT,ADVERTISER_ID,CAMPAIGN_ID,CREATE_TIME,ADGROUP_NAME,PLACEMENT_TYPE,PROFILE_IMAGE,LANDING_PAGE_URL,DISPLAY_NAME,APP_TYPE,APP_DOWNLOAD_URL,APP_NAME,OPTIMIZATION_EVENT,SECONDARY_OPTIMIZATION_EVENT,CREATIVE_MATERIAL_MODE,AUDIENCE_TYPE,GENDER,MIN_ANDROID_VERSION,MIN_IOS_VERSION,BUDGET_MODE,SCHEDULE_TYPE,DAYPARTING,OPTIMIZATION_GOAL,CPV_VIDEO_DURATION,PACING,BILLING_EVENT,BID_TYPE,DEEP_BID_TYPE,IMPRESSION_TRACKING_URL,CLICK_TRACKING_URL,SECONDARY_STATUS,OPERATION_STATUS,STATISTIC_TYPE,VIDEO_DOWNLOAD,OPEN_URL,OPEN_URL_TYPE,FALLBACK_TYPE,BUDGET,BID_PRICE,CONVERSION_BID_PRICE,DEEP_CPA_BID,SCHEDULE_START_TIME,SCHEDULE_END_TIME,APP_ID,PIXEL_ID,INVENTORY_FILTER_ENABLED,IS_HFSS,IS_NEW_STRUCTURE,CATEGORY,IS_COMMENT_DISABLE,SKIP_LEARNING_PHASE,FREQUENCY,FREQUENCY_SCHEDULE,ACTION_DAYS,AUDIENCE,EXCLUDED_AUDIENCE,LOCATION,INTEREST_CATEGORY_V_2,PANGLE_BLOCK_APP_LIST_ID,ACTION_CATEGORIES,PLACEMENTS,KEYWORDS,AGE_GROUPS,LANGUAGES,OPERATING_SYSTEMS,NETWORK_TYPES,CARRIERS,VIDEO_ACTIONS,PACKAGE,_FIVETRAN_SYNCED
 1,2021-09-15 02:44:56.000,111,222,2021-08-05 01:34:25.000,Ad group name one,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,DYNAMIC,,GENDER_UNLIMITED,,,BUDGET_MODE_DAY,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,500,0,0,0,2021-08-05 02:30:23.000,2031-08-03 02:30:23.000,,543543543,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.652
 2,2021-09-15 00:52:08.000,222,333,2021-08-16 20:40:16.000,Ad group name two,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,DYNAMIC,,GENDER_UNLIMITED,,,BUDGET_MODE_INFINITE,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,0,0,0,0,2021-08-17 13:00:08.000,2031-08-15 13:00:08.000,,543543543,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.643
 3,2021-09-15 02:44:55.000,333,444,2021-09-13 18:44:14.000,Ad group name three,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,CUSTOM,,GENDER_UNLIMITED,,,BUDGET_MODE_DAY,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,900,0,0,0,2021-09-13 18:44:05.000,2031-09-11 18:44:05.000,,654645654,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.635

integration_tests/seeds/tiktok_advertiser_data.csv

@@ -1,3 +1,3 @@
-ID,NAME,ADDRESS,COMPANY,CONTACTER,COUNTRY,CURRENCY,DESCRIPTION,EMAIL,INDUSTRY,LICENSE_NO,LICENSE_URL,PROMOTION_AREA,REASON,ROLE,STATUS,TELEPHONE,TIMEZONE,BALANCE,CREATE_TIME,LANGUAGE,PHONE_NUMBER,_FIVETRAN_SYNCED
-1,US,1600 pennsylvania avenue,White House,,US,USD,hello,louis@vuitton.com,123,,,111,,ROLE_ADVERTISER,STATUS_ENABLE,1,Etc/GMT+5,32.12,2021-08-27 18:13:41.000,en,string,2021-09-15 15:53:18.938
-2,Canada,21 w wacker st,Blanco House,,CAN,CAN,world,jeff@bezos.com,321,,,0,,ROLE_ADVERTISER,STATUS_ENABLE,1,Etc/GMT+5,321.22,2021-03-09 20:55:37.000,en,string,2021-09-15 15:53:07.621
+ID,NAME,ADDRESS,COMPANY,CONTACTER,COUNTRY,CURRENCY,DESCRIPTION,EMAIL,INDUSTRY,LICENSE_NO,LICENSE_URL,PROMOTION_AREA,REJECTED_REASON,ROLE,STATUS,TELEPHONE_NUMBER,TIMEZONE,BALANCE,CREATE_TIME,LANGUAGE,CELLPHONE_NUMBER,_FIVETRAN_SYNCED
+1,US,1600 pennsylvania avenue,White House,,US,USD,hello,louis@vuitton.com,123,,,111,,ROLE_ADVERTISER,STATUS_ENABLE,111-222-3333,Etc/GMT+5,32.12,2021-08-27 18:13:41.000,en,001-002-0003,2021-09-15 15:53:18.938
+2,Canada,21 w wacker st,Blanco House,,CAN,CAN,world,jeff@bezos.com,321,,,0,,ROLE_ADVERTISER,STATUS_ENABLE,222-333-4444,Etc/GMT+5,321.22,2021-03-09 20:55:37.000,en,001-003-0004,2021-09-15 15:53:07.621

integration_tests/seeds/tiktok_campaign_history_data.csv

@@ -1,5 +1,5 @@
-CAMPAIGN_ID,UPDATED_AT,ADVERTISER_ID,CAMPAIGN_NAME,CAMPAIGN_TYPE,BUDGET,BUDGET_MODE,OPT_STATUS,OBJECTIVE_TYPE,IS_NEW_STRUCTURE,SPLIT_TEST_VARIABLE,CREATE_TIME,_FIVETRAN_SYNCED
-1,2021-09-15 00:52:08.000,1,campaign name one,REGULAR_CAMPAIGN,450,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:40:15.000,2021-09-15 02:47:54.737
-2,2021-09-15 01:33:37.000,2,campaign name two,REGULAR_CAMPAIGN,300,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:29:06.000,2021-09-15 02:47:54.737
-3,2021-09-14 02:53:48.000,3,campaign name three,REGULAR_CAMPAIGN,450,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:40:15.000,2021-09-14 03:01:03.005
-4,2021-09-14 02:59:01.000,4,campaign name four,REGULAR_CAMPAIGN,300,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:29:06.000,2021-09-14 03:01:03.006
+AD_ID,UPDATED_AT,ADVERTISER_ID,ADGROUP_ID,CAMPAIGN_ID,CREATE_TIME,AD_NAME,CALL_TO_ACTION,SECONDARY_STATUS,OPERATION_STATUS,AD_TEXT,VIDEO_ID,APP_NAME,DEEPLINK,LANDING_PAGE_URL,DISPLAY_NAME,PROFILE_IMAGE_URL,IMPRESSION_TRACKING_URL,CLICK_TRACKING_URL,PLAYABLE_URL,IS_ACO,CREATIVE_AUTHORIZED,IS_NEW_STRUCTURE,IMAGE_IDS,_FIVETRAN_SYNCED
+1,2021-09-15 04:26:28.000,1,1,1,2021-08-27 14:13:57.000,ad name one,SIGN_UP,AD_STATUS_DELIVERY_OK,DISABLE,subscribe,v10033g50000c4kf54jc77ucvou6fv0g,,,https://www.stuffthatworks.health/,health,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:16.797
+2,2021-09-15 03:19:25.000,2,2,2,2021-09-13 18:54:11.000,ad name two,SIGN_UP,AD_STATUS_DELIVERY_OK,ENABLE,join us,v10033g50000c4vnvmbc77u58bhlklvg,,,https://swaggystocks.com/dashboard/home,stocks,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:16.641
+3,2021-09-15 03:54:51.000,3,3,3,2021-07-26 16:28:39.000,ad name three,INTERESTED,AD_STATUS_DELIVERY_OK,ENABLE,like and share,v10033g50000c3qbo6jc77uaacd9hvv0,,,https://finviz.com/screener.ashx,more stocks,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:18.918
+4,2021-09-15 04:26:26.000,4,4,4,2021-08-05 21:02:33.000,ad name four,SUBSCRIBE,AD_STATUS_DELIVERY_OK,DISABLE,get stuck in,v10033g50000c3qbo6jc77ucp113o3qg,,,linkedin.com,linkedin,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:18.897

models/tiktok_ads.yml

@@ -94,22 +94,14 @@ models:
         description: Ad group ID.
       - name: ad_group_name
         description: Ad group name.
-      - name: action_categories
-        description: IDs of the action categories (behaviors) that you want to target.
-      - name: age
-        description: Age groups you want to target. 
       - name: audience_type
         description: Audience Type
       - name: budget
         description: Ad budget. Returns 0.0 when Campaign Budget Optimization (budget_optimize_switch) is on.
       - name: category
         description: Ad group category.
-      - name: interest_category
-        description: Interest classification. If the interest is specified, users that do not meet interest target will be excluded during delivery.
       - name: gender
         description: Gender that you want to target.
-      - name: languages
-        description: Codes of the languages that you want to target.
       - name: currency
         description: Advertiser's currency.
       - name: impressions
@@ -182,22 +174,14 @@ models:
         description: Advertiser name.
       - name: currency
         description: Advertiser's currency.
-      - name: action_categories
-        description: IDs of the action categories (behaviors) that you want to target.
-      - name: age
-        description: Age groups you want to target. 
       - name: audience_type
         description: Audience Type
       - name: budget
         description: Ad budget. Returns 0.0 when Campaign Budget Optimization (budget_optimize_switch) is on.
       - name: category
         description: Ad group category.
-      - name: interest_category
-        description: Interest classification. If the interest is specified, users that do not meet interest target will be excluded during delivery.
       - name: gender
         description: Gender that you want to target.
-      - name: languages
-        description: Codes of the languages that you want to target.
       - name: impressions
         description: The number of impressions that occurred on the day of the record.
       - name: clicks
@@ -360,22 +344,14 @@ models:
         description: The utm_content parameter of the ad, extracted from the `landing page url`.
       - name: utm_term
         description: The utm_term parameter of the ad, extracted from the `landing page url`.
-      - name: action_categories
-        description: IDs of the action categories (behaviors) that you want to target.
-      - name: age
-        description: Age groups you want to target. 
       - name: audience_type
         description: Audience Type
       - name: budget
         description: Ad budget. Returns 0.0 when Campaign Budget Optimization (budget_optimize_switch) is on.
       - name: category
         description: Ad group category.
-      - name: interest_category
-        description: Interest classification. If the interest is specified, users that do not meet interest target will be excluded during delivery.
       - name: gender
         description: Gender that you want to target.
-      - name: languages
-        description: Codes of the languages that you want to target.
       - name: currency
         description: Advertiser's currency.
       - name: impressions

models/tiktok_ads__ad_group_report.sql

@@ -36,15 +36,11 @@ aggregated as (
         campaigns.campaign_name,
         hourly.ad_group_id,
         ad_groups.ad_group_name,
-        advertiser.currency,
-        ad_groups.action_categories,
+        advertiser.currency, 
         ad_groups.category,
         ad_groups.gender,
         ad_groups.audience_type,
         ad_groups.budget,
-        ad_groups.age,
-        ad_groups.languages,
-        ad_groups.interest_category,
         sum(hourly.impressions) as impressions,
         sum(hourly.clicks) as clicks,
         sum(hourly.spend) as spend,
@@ -73,7 +69,7 @@ aggregated as (
         on ad_groups.advertiser_id = advertiser.advertiser_id
     left join campaigns
         on ad_groups.campaign_id = campaigns.campaign_id
-    {{ dbt_utils.group_by(16) }}
+    {{ dbt_utils.group_by(12) }}
 
 )
 

models/tiktok_ads__ad_report.sql

@@ -47,13 +47,9 @@ aggregated as (
         ads.ad_name,
         advertiser.currency,
         ad_groups.category,
-        ad_groups.action_categories,
         ad_groups.gender,
         ad_groups.audience_type,
         ad_groups.budget,
-        ad_groups.age,
-        ad_groups.languages,
-        ad_groups.interest_category,
         sum(hourly.impressions) as impressions,
         sum(hourly.clicks) as clicks,
         sum(hourly.spend) as spend,
@@ -84,7 +80,7 @@ aggregated as (
         on ads.advertiser_id = advertiser.advertiser_id
     left join campaigns
         on ads.campaign_id = campaigns.campaign_id
-    {{ dbt_utils.group_by(18) }}
+    {{ dbt_utils.group_by(14) }}
 
 )
 

models/tiktok_ads__url_report.sql

@@ -54,14 +54,10 @@ aggregated as (
         ads.utm_content,
         ads.utm_term,
         advertiser.currency,
-        ad_groups.action_categories,
         ad_groups.category,
         ad_groups.gender,
         ad_groups.audience_type,
         ad_groups.budget,
-        ad_groups.age,
-        ad_groups.languages,
-        ad_groups.interest_category,
         sum(hourly.impressions) as impressions,
         sum(hourly.clicks) as clicks,
         sum(hourly.spend) as spend,
@@ -98,7 +94,7 @@ aggregated as (
         where ads.landing_page_url is not null
     {% endif %}
 
-    {{ dbt_utils.group_by(26) }}
+    {{ dbt_utils.group_by(22) }}
 
 )
 

packages.yml

@@ -1,3 +1,7 @@
 packages:
-- package: fivetran/tiktok_ads_source
-  version: [">=0.3.0", "<0.4.0"]
+# - package: fivetran/tiktok_ads_source
+#   version: [">=0.3.0", "<0.4.0"]
+
+  - git: https://github.com/fivetran/dbt_tiktok_ads_source.git
+    revision: feature/tiktok-ads-v1.3-api-update
+    warn-unpinned: false