Merge pull request #10 from fivetran/feature/tiktok-ads-v1.3-api-update

Version 1.3 API updates
fivetran · Jul 25, 2023 · 5ccb503 · 5ccb503
2 parents f10d5a1 + e9d6df2
commit 5ccb503
Show file tree

Hide file tree

Showing 20 changed files with 62 additions and 30 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,5 @@ target/
 dbt_modules/
 logs/
 dbt_packages/
+env/
 .DS_Store
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,9 +1,16 @@
-# dbt_tiktok_ads_source v0.UPDATE.UPDATE
-
- ## Under the Hood:
-
+# dbt_tiktok_ads_source v0.4.0
+[PR #10](https://github.com/fivetran/dbt_tiktok_ads_source/pull/10) applies the following updates:
+## 🚨 Breaking Changes 🚨
+- In the [July 2023 connector update for TikTok Ads](https://fivetran.com/docs/applications/tiktok-ads/changelog), we upgraded the connector to the v1.3 API. As a result the dependent fields and field names from the downstream staging models have changed.
+  - In `stg_tiktok_ads__ad_group_history`, `age` has been renamed as `age_groups`.
+  - In `stg_tiktok_ads__advertiser`, `phone_number` has been renamed as `cellphone_number`, and `telephone` has been renamed as `telephone_number`.
+- **Note**: We coalesced the old API field names with these renamed fields to ensure backwards compatibility for these fields. 
+
+## 🔧 Under the Hood 🔩
 - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
 - Updated the pull request [templates](/.github).
+- Seed files were also renamed with the newest version of the fields, in case we decide to bring them in with future versions of the package.
+
 # dbt_tiktok_ads_source v0.3.0
 
 ## 🚨 Breaking Changes 🚨:

diff --git a/README.md b/README.md
@@ -40,7 +40,7 @@ If you  are **not** using the [Tiktok transformation package](https://github.com
 ```yaml
 packages:
   - package: fivetran/tiktok_ads_source
-    version: [">=0.3.0", "<0.4.0"]
+    version: [">=0.4.0", "<0.5.0"]
 ```
 
 ## Step 3: Define database and schema variables

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'tiktok_ads_source'
-version: '0.3.0'
+version: '0.4.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'tiktok_ads_source_integration_tests'
-version: '0.3.0'
+version: '0.4.0'
 profile: 'integration_tests'
 config-version: 2
 

diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml
@@ -1,2 +1,2 @@
 packages:
- - local: ../
+  - local: ../
diff --git a/integration_tests/seeds/tiktok_ad_history_data.csv b/integration_tests/seeds/tiktok_ad_history_data.csv
@@ -1,4 +1,4 @@
-AD_ID,UPDATED_AT,ADVERTISER_ID,ADGROUP_ID,CAMPAIGN_ID,CREATE_TIME,AD_NAME,CALL_TO_ACTION,STATUS,OPT_STATUS,AD_TEXT,VIDEO_ID,APP_NAME,OPEN_URL,LANDING_PAGE_URL,DISPLAY_NAME,PROFILE_IMAGE,IMPRESSION_TRACKING_URL,CLICK_TRACKING_URL,PLAYABLE_URL,IS_ACO,IS_CREATIVE_AUTHORIZED,IS_NEW_STRUCTURE,IMAGE_IDS,_FIVETRAN_SYNCED
+AD_ID,UPDATED_AT,ADVERTISER_ID,ADGROUP_ID,CAMPAIGN_ID,CREATE_TIME,AD_NAME,CALL_TO_ACTION,SECONDARY_STATUS,OPERATION_STATUS,AD_TEXT,VIDEO_ID,APP_NAME,DEEPLINK,LANDING_PAGE_URL,DISPLAY_NAME,PROFILE_IMAGE_URL,IMPRESSION_TRACKING_URL,CLICK_TRACKING_URL,PLAYABLE_URL,IS_ACO,CREATIVE_AUTHORIZED,IS_NEW_STRUCTURE,IMAGE_IDS,_FIVETRAN_SYNCED
 1,2021-09-15 04:26:28.000,1,1,1,2021-08-27 14:13:57.000,ad name one,SIGN_UP,AD_STATUS_DELIVERY_OK,DISABLE,subscribe,v10033g50000c4kf54jc77ucvou6fv0g,,,https://www.stuffthatworks.health/,health,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:16.797
 2,2021-09-15 03:19:25.000,2,2,2,2021-09-13 18:54:11.000,ad name two,SIGN_UP,AD_STATUS_DELIVERY_OK,ENABLE,join us,v10033g50000c4vnvmbc77u58bhlklvg,,,https://swaggystocks.com/dashboard/home,stocks,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:16.641
 3,2021-09-15 03:54:51.000,3,3,3,2021-07-26 16:28:39.000,ad name three,INTERESTED,AD_STATUS_DELIVERY_OK,ENABLE,like and share,v10033g50000c3qbo6jc77uaacd9hvv0,,,https://finviz.com/screener.ashx,more stocks,,,,,TRUE,FALSE,TRUE,[ 124938917 ],2021-09-15 15:53:18.918

diff --git a/integration_tests/seeds/tiktok_adgroup_history_data.csv b/integration_tests/seeds/tiktok_adgroup_history_data.csv
@@ -1,4 +1,4 @@
-ADGROUP_ID,UPDATED_AT,ADVERTISER_ID,CAMPAIGN_ID,CREATE_TIME,ADGROUP_NAME,PLACEMENT_TYPE,PROFILE_IMAGE,LANDING_PAGE_URL,DISPLAY_NAME,APP_TYPE,APP_DOWNLOAD_URL,APP_NAME,EXTERNAL_ACTION,DEEP_EXTERNAL_ACTION,CREATIVE_MATERIAL_MODE,AUDIENCE_TYPE,GENDER,ANDROID_OSV,IOS_OSV,BUDGET_MODE,SCHEDULE_TYPE,DAYPARTING,OPTIMIZE_GOAL,CPV_VIDEO_DURATION,PACING,BILLING_EVENT,BID_TYPE,DEEP_BID_TYPE,IMPRESSION_TRACKING_URL,CLICK_TRACKING_URL,STATUS,OPT_STATUS,STATISTIC_TYPE,VIDEO_DOWNLOAD,OPEN_URL,OPEN_URL_TYPE,FALLBACK_TYPE,BUDGET,BID,CONVERSION_BID,DEEP_CPABID,SCHEDULE_START_TIME,SCHEDULE_END_TIME,APP_ID,PIXEL_ID,ENABLE_INVENTORY_FILTER,IS_HFSS,IS_NEW_STRUCTURE,CATEGORY,IS_COMMENT_DISABLE,SKIP_LEARNING_PHASE,FREQUENCY,FREQUENCY_SCHEDULE,ACTION_DAYS,AUDIENCE,EXCLUDED_AUDIENCE,LOCATION,INTEREST_CATEGORY_V_2,PANGLE_BLOCK_APP_LIST_ID,ACTION_CATEGORIES,PLACEMENT,KEYWORDS,AGE,LANGUAGES,OPERATION_SYSTEM,CONNECTION_TYPE,CARRIERS,VIDEO_ACTIONS,PACKAGE,_FIVETRAN_SYNCED
+ADGROUP_ID,UPDATED_AT,ADVERTISER_ID,CAMPAIGN_ID,CREATE_TIME,ADGROUP_NAME,PLACEMENT_TYPE,PROFILE_IMAGE,LANDING_PAGE_URL,DISPLAY_NAME,APP_TYPE,APP_DOWNLOAD_URL,APP_NAME,OPTIMIZATION_EVENT,SECONDARY_OPTIMIZATION_EVENT,CREATIVE_MATERIAL_MODE,AUDIENCE_TYPE,GENDER,MIN_ANDROID_VERSION,MIN_IOS_VERSION,BUDGET_MODE,SCHEDULE_TYPE,DAYPARTING,OPTIMIZATION_GOAL,CPV_VIDEO_DURATION,PACING,BILLING_EVENT,BID_TYPE,DEEP_BID_TYPE,IMPRESSION_TRACKING_URL,CLICK_TRACKING_URL,SECONDARY_STATUS,OPERATION_STATUS,STATISTIC_TYPE,VIDEO_DOWNLOAD,OPEN_URL,OPEN_URL_TYPE,FALLBACK_TYPE,BUDGET,BID_PRICE,CONVERSION_BID_PRICE,DEEP_CPA_BID,SCHEDULE_START_TIME,SCHEDULE_END_TIME,APP_ID,PIXEL_ID,INVENTORY_FILTER_ENABLED,IS_HFSS,IS_NEW_STRUCTURE,CATEGORY,IS_COMMENT_DISABLE,SKIP_LEARNING_PHASE,FREQUENCY,FREQUENCY_SCHEDULE,ACTION_DAYS,AUDIENCE,EXCLUDED_AUDIENCE,LOCATION,INTEREST_CATEGORY_V_2,PANGLE_BLOCK_APP_LIST_ID,ACTION_CATEGORIES,PLACEMENTS,KEYWORDS,AGE_GROUPS,LANGUAGES,OPERATING_SYSTEMS,NETWORK_TYPES,CARRIERS,VIDEO_ACTIONS,PACKAGE,_FIVETRAN_SYNCED
 1,2021-09-15 02:44:56.000,111,222,2021-08-05 01:34:25.000,Ad group name one,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,DYNAMIC,,GENDER_UNLIMITED,,,BUDGET_MODE_DAY,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,500,0,0,0,2021-08-05 02:30:23.000,2031-08-03 02:30:23.000,,543543543,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.652
 2,2021-09-15 00:52:08.000,222,333,2021-08-16 20:40:16.000,Ad group name two,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,DYNAMIC,,GENDER_UNLIMITED,,,BUDGET_MODE_INFINITE,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,0,0,0,0,2021-08-17 13:00:08.000,2031-08-15 13:00:08.000,,543543543,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.643
 3,2021-09-15 02:44:55.000,333,444,2021-09-13 18:44:14.000,Ad group name three,PLACEMENT_TYPE_NORMAL,,,,,,,ON_WEB_REGISTER,,CUSTOM,,GENDER_UNLIMITED,,,BUDGET_MODE_DAY,SCHEDULE_FROM_NOW,,CONVERT,,PACING_MODE_SMOOTH,OCPM,BID_TYPE_NO_BID,,,,ADGROUP_STATUS_DELIVERY_OK,ENABLE,,ALLOW_DOWNLOAD,,,,900,0,0,0,2021-09-13 18:44:05.000,2031-09-11 18:44:05.000,,654645654,FALSE,FALSE,TRUE,0,1,1,,,,[ 124938917 ],[ 124938917 ],[ 124938917 ],[ 124938917 ],[],[],"[ ""PLACEMENT_TIKTOK"" ]",,"[ ""AGE_25_34"", ""AGE_35_44"", ""AGE_45_54"", ""AGE_55_100"", ""AGE_18_24"" ]","[ ""en"" ]",[],[],[],[],,2021-09-15 02:47:55.635

diff --git a/integration_tests/seeds/tiktok_advertiser_data.csv b/integration_tests/seeds/tiktok_advertiser_data.csv
@@ -1,3 +1,3 @@
-ID,NAME,ADDRESS,COMPANY,CONTACTER,COUNTRY,CURRENCY,DESCRIPTION,EMAIL,INDUSTRY,LICENSE_NO,LICENSE_URL,PROMOTION_AREA,REASON,ROLE,STATUS,TELEPHONE,TIMEZONE,BALANCE,CREATE_TIME,LANGUAGE,PHONE_NUMBER,_FIVETRAN_SYNCED
-1,US,1600 pennsylvania avenue,White House,,US,USD,hello,louis@vuitton.com,123,,,111,,ROLE_ADVERTISER,STATUS_ENABLE,1,Etc/GMT+5,32.12,2021-08-27 18:13:41.000,en,string,2021-09-15 15:53:18.938
-2,Canada,21 w wacker st,Blanco House,,CAN,CAN,world,jeff@bezos.com,321,,,0,,ROLE_ADVERTISER,STATUS_ENABLE,1,Etc/GMT+5,321.22,2021-03-09 20:55:37.000,en,string,2021-09-15 15:53:07.621
+ID,NAME,ADDRESS,COMPANY,CONTACTER,COUNTRY,CURRENCY,DESCRIPTION,EMAIL,INDUSTRY,LICENSE_NO,LICENSE_URL,PROMOTION_AREA,REJECTED_REASON,ROLE,STATUS,TELEPHONE_NUMBER,TIMEZONE,BALANCE,CREATE_TIME,LANGUAGE,CELLPHONE_NUMBER,_FIVETRAN_SYNCED
+1,US,1600 pennsylvania avenue,White House,,US,USD,hello,louis@vuitton.com,123,,,111,,ROLE_ADVERTISER,STATUS_ENABLE,111-222-3333,Etc/GMT+5,32.12,2021-08-27 18:13:41.000,en,001-002-0003,2021-09-15 15:53:18.938
+2,Canada,21 w wacker st,Blanco House,,CAN,CAN,world,jeff@bezos.com,321,,,0,,ROLE_ADVERTISER,STATUS_ENABLE,222-333-4444,Etc/GMT+5,321.22,2021-03-09 20:55:37.000,en,001-003-0004,2021-09-15 15:53:07.621
diff --git a/integration_tests/seeds/tiktok_campaign_history_data.csv b/integration_tests/seeds/tiktok_campaign_history_data.csv
@@ -1,4 +1,4 @@
-CAMPAIGN_ID,UPDATED_AT,ADVERTISER_ID,CAMPAIGN_NAME,CAMPAIGN_TYPE,BUDGET,BUDGET_MODE,OPT_STATUS,OBJECTIVE_TYPE,IS_NEW_STRUCTURE,SPLIT_TEST_VARIABLE,CREATE_TIME,_FIVETRAN_SYNCED
+CAMPAIGN_ID,UPDATED_AT,ADVERTISER_ID,CAMPAIGN_NAME,CAMPAIGN_TYPE,BUDGET,BUDGET_MODE,OPERATION_STATUS,OBJECTIVE_TYPE,IS_NEW_STRUCTURE,SPLIT_TEST_VARIABLE,CREATE_TIME,_FIVETRAN_SYNCED
 1,2021-09-15 00:52:08.000,1,campaign name one,REGULAR_CAMPAIGN,450,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:40:15.000,2021-09-15 02:47:54.737
 2,2021-09-15 01:33:37.000,2,campaign name two,REGULAR_CAMPAIGN,300,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:29:06.000,2021-09-15 02:47:54.737
 3,2021-09-14 02:53:48.000,3,campaign name three,REGULAR_CAMPAIGN,450,BUDGET_MODE_DAY,ENABLE,CONVERSIONS,TRUE,,2021-08-16 20:40:15.000,2021-09-14 03:01:03.005

diff --git a/macros/get_ad_group_history_columns.sql b/macros/get_ad_group_history_columns.sql
@@ -18,6 +18,7 @@
     {"name": "interest_category_v_2", "datatype": dbt.type_string()},
     {"name": "action_categories", "datatype": dbt.type_string()},
     {"name": "age", "datatype": dbt.type_string()},
+    {"name": "age_groups", "datatype": dbt.type_string()},
     {"name": "languages", "datatype": dbt.type_string()}
 
 ] %}

diff --git a/macros/get_advertiser_columns.sql b/macros/get_advertiser_columns.sql
@@ -3,6 +3,7 @@
 {% set columns = [
     {"name": "address", "datatype": dbt.type_string()},
     {"name": "balance", "datatype": dbt.type_float()},
+    {"name": "cellphone_number", "datatype": dbt.type_string()},
     {"name": "company", "datatype": dbt.type_string()},
     {"name": "contacter", "datatype": dbt.type_string()},
     {"name": "country", "datatype": dbt.type_string()},
@@ -15,6 +16,7 @@
     {"name": "name", "datatype": dbt.type_string()},
     {"name": "phone_number", "datatype": dbt.type_string()},
     {"name": "telephone", "datatype": dbt.type_string()},
+    {"name": "telephone_number", "datatype": dbt.type_string()},
     {"name": "timezone", "datatype": dbt.type_string()}
 ] %}
 

diff --git a/models/src_tiktok_ads.yml b/models/src_tiktok_ads.yml
@@ -26,6 +26,10 @@ sources:
             description: Advertiser address information
           - name: balance
             description: Account available balance
+          - name: cellphone_number
+            description: > 
+              Contact mobile number, desensitised data.
+              This is the newest version of the field `phone_number`, which was renamed after the Tiktok Ads v1.3 API release.
           - name: company
             description: Advertiser's company name
           - name: contacter
@@ -51,7 +55,10 @@ sources:
           - name: name
             description: Advertiser name
           - name: phone_number
-            description: Contact mobile number, desensitised data
+            description: > 
+              Contact mobile number, desensitised data.
+              IMPORTANT: This field will not be populated for connectors utilizing the Tiktok Ads v1.3 API version, as the column was renamed. 
+              The new column name is `cellphone_number`.
           - name: promotion_area
             description: Operation area
           - name: reason 
@@ -61,7 +68,14 @@ sources:
           - name: status
             description: Advertiser status
           - name: telephone
-            description: Fixed phone number, desensitised data
+            description: >
+              Fixed phone number, desensitised data
+              IMPORTANT: This field will not be populated for connectors utilizing the Tiktok Ads v1.3 API version, as the column was renamed. 
+              The new column name is `telephone_number`.
+          - name: telephone_number
+            description: >
+              Fixed phone number, desensitised data
+              This is the newest version of the field `telephone`, which was renamed after the Tiktok Ads v1.3 API release.
           - name: timezone
             description: Ad account time zone including GMT offset
           - name: _fivetran_synced
@@ -117,7 +131,14 @@ sources:
           - name: adgroup_name
             description: Ad group name. Character limit is 512 and cannot contain emoji.
           - name: age
-            description: Age groups you want to target. 
+            description: >
+              Age groups you want to target. 
+              IMPORTANT: This field will not be populated for connectors utilizing the Tiktok Ads v1.3 API version, as the column was renamed. 
+              The new column name is `age_group`.
+          - name: age_groups
+            description: >
+              Age groups you want to target.
+              This is the newest version of the field `age`, which was renamed after the Tiktok Ads v1.3 API release.
           - name: android_osv
             description: Minimum Android version. 
           - name: app_download_url

diff --git a/models/stg_tiktok_ads.yml b/models/stg_tiktok_ads.yml
@@ -39,9 +39,9 @@ models:
         description: License preview address, the link is valid for an hour by default.
       - name: advertiser_name
         description: Advertiser name
-      - name: phone_number
+      - name: cellphone_number
         description: Contact mobile number, desensitised data
-      - name: telephone
+      - name: telephone_number
         description: Fixed phone number, desensitised data
       - name: timezone
         description: Ad account time zone including GMT offset
@@ -71,7 +71,7 @@ models:
         description: Action days
       - name: ad_group_name
         description: Ad group name. Character limit is 512 and cannot contain emoji.
-      - name: age
+      - name: age_groups
         description: Age groups you want to target. 
       - name: audience_type
         description: Audience Type

diff --git a/models/stg_tiktok_ads__ad_group_history.sql b/models/stg_tiktok_ads__ad_group_history.sql
@@ -29,7 +29,7 @@ final as (
         action_days,
         action_categories,
         adgroup_name as ad_group_name,
-        age,
+        coalesce(age_groups, age) as age_groups,
         audience_type,
         budget,
         category,

diff --git a/models/stg_tiktok_ads__advertiser.sql b/models/stg_tiktok_ads__advertiser.sql
@@ -34,8 +34,8 @@ final as (
         industry, 
         language,
         name as advertiser_name, 
-        phone_number, 
-        telephone, 
+        coalesce(cellphone_number, phone_number) as cellphone_number, 
+        coalesce(telephone_number, telephone) as telephone_number,
         timezone
     from fields
 )