✨ Source TikTok Marketing: add new streams (airbytehq#31610)

Co-authored-by: davydov-d <davydov-d@users.noreply.github.com>
ariesgun · Oct 20, 2023 · 5af98da · 5af98da
1 parent d2fb8d9
commit 5af98da
Show file tree

Hide file tree

Showing 17 changed files with 445 additions and 26 deletions.
diff --git a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml
@@ -20,7 +20,7 @@ data:
   name: Paypal Transaction
   registries:
     cloud:
-      dockerImageTag: 2.0.0  #https://github.com/airbytehq/oncall/issues/3347
+      dockerImageTag: 2.0.0 #https://github.com/airbytehq/oncall/issues/3347
       enabled: true
     oss:
       enabled: true

diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml
@@ -50,6 +50,16 @@ acceptance_tests:
           ads:
             - name: "profile_image_url"
               bypass_reason: "Volatile URLs"
+          creative_assets_images:
+            - name: "image_url"
+              bypass_reason: "Volatile URL params"
+          creative_assets_videos:
+            - name: "preview_url"
+              bypass_reason: "Volatile URL params"
+            - name: "video_cover_url"
+              bypass_reason: "Volatile URL params"
+            - name: "preview_url_expire_time"
+              bypass_reason: "Changes over time"
         empty_streams:
           - name: ads_reports_hourly
             bypass_reason: "Tested with daily granularity."
@@ -75,6 +85,10 @@ acceptance_tests:
             bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on."
           - name: ads_audience_reports_by_platform_daily
             bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on."
+          - name: creative_assets_portfolios
+            bypass_reason: "No data in the integration test account. We should seed the sandbox later on."
+          - name: creative_assets_music
+            bypass_reason: "System music provided by TikTok - very volatile data."
         # Old style streams with granularity config (for < 0.1.13)
         # Note: not needed to be tested separately in full and incremental tests, because code of
         # these streams is called directly in new style streams
@@ -85,6 +99,20 @@ acceptance_tests:
           exact_order: no
           extra_records: yes
         timeout_seconds: 1200
+        ignored_fields:
+          ads:
+            - name: "profile_image_url"
+              bypass_reason: "Volatile URLs"
+          creative_assets_images:
+            - name: "image_url"
+              bypass_reason: "Volatile URL params"
+          creative_assets_videos:
+            - name: "preview_url"
+              bypass_reason: "Volatile URL params"
+            - name: "video_cover_url"
+              bypass_reason: "Volatile URL params"
+            - name: "preview_url_expire_time"
+              bypass_reason: "Changes over time"
         empty_streams:
           - name: ad_groups
             bypass_reason: "This stream is tested on the new style config."
@@ -96,6 +124,10 @@ acceptance_tests:
             bypass_reason: "This stream is tested on the new style config."
           - name: campaigns
             bypass_reason: "This stream is tested on the new style config."
+          - name: creative_assets_portfolios
+            bypass_reason: "No data in the integration test account. We should seed the sandbox later on."
+          - name: creative_assets_music
+            bypass_reason: "System music provided by TikTok - very volatile data."
   full_refresh:
     tests:
       - config_path: "secrets/config.json"
@@ -105,6 +137,16 @@ acceptance_tests:
           ads:
             - name: "profile_image_url"
               bypass_reason: "Volatile URLs"
+          creative_assets_images:
+            - name: "image_url"
+              bypass_reason: "Volatile URL params"
+          creative_assets_videos:
+            - name: "preview_url"
+              bypass_reason: "Volatile URL params"
+            - name: "video_cover_url"
+              bypass_reason: "Volatile URL params"
+            - name: "preview_url_expire_time"
+              bypass_reason: "Changes over time"
   incremental:
     tests:
       - config_path: "secrets/config.json"

diff --git a/...yte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json b/...yte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json
@@ -119,5 +119,27 @@
         "modify_time": "2099-01-01 01:00:0"
       }
     }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_descriptor": {
+        "name": "creative_assets_images"
+      },
+      "stream_state": {
+        "modify_time": "2099-01-01 01:00:0"
+      }
+    }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_descriptor": {
+        "name": "creative_assets_videos"
+      },
+      "stream_state": {
+        "modify_time": "2099-01-01 01:00:0"
+      }
+    }
   }
 ]
diff --git a/...integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json b/...integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json
@@ -183,6 +183,56 @@
       "destination_sync_mode": "overwrite",
       "cursor_field": ["stat_time_day"],
       "primary_key": [["advertiser_id"], ["stat_time_day"], ["gender"], ["age"]]
+    },
+    {
+      "stream": {
+        "name": "audiences",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["audience_id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["audience_id"]]
+    },
+    {
+      "stream": {
+        "name": "creative_assets_portfolios",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["creative_portfolio_id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["creative_portfolio_id"]]
+    },
+    {
+      "stream": {
+        "name": "creative_assets_images",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["modify_time"],
+        "source_defined_primary_key": [["image_id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["modify_time"],
+      "primary_key": [["image_id"]]
+    },
+    {
+      "stream": {
+        "name": "creative_assets_videos",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["modify_time"],
+        "source_defined_primary_key": [["video_id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "cursor_field": ["modify_time"],
+      "primary_key": [["video_id"]]
     }
   ]
 }
diff --git a/...-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl b/...-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl
@@ -48,3 +48,11 @@
 {"stream": "advertisers_audience_reports_lifetime", "data": {"dimensions": {"gender": "MALE", "age": "AGE_35_44", "advertiser_id": 7002238017842757633}, "metrics": {"ctr": "1.26", "cpc": "0.36", "spend": "31.56", "cpm": "4.58", "clicks": "87", "impressions": "6897"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1697639275347}
 {"stream": "advertisers_audience_reports_lifetime", "data": {"dimensions": {"gender": "FEMALE", "age": "AGE_55_100", "advertiser_id": 7002238017842757633}, "metrics": {"ctr": "0.00", "cpc": "0.00", "spend": "0.00", "cpm": "0.00", "clicks": "0", "impressions": "17"}, "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_55_100"}, "emitted_at": 1697639275356}
 {"stream": "advertisers_audience_reports_lifetime", "data": {"dimensions": {"gender": "MALE", "age": "AGE_13_17", "advertiser_id": 7002238017842757633}, "metrics": {"ctr": "1.21", "cpc": "0.32", "spend": "7.13", "cpm": "3.92", "clicks": "22", "impressions": "1818"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1697639275364}
+{"stream": "audiences", "data": {"is_valid": false, "audience_type": "Lead Generation", "expired_time": "2022-10-20 07:26:39", "create_time": "2021-10-20 07:26:39", "is_expiring": true, "cover_num": 0, "shared": false, "is_creator": true, "calculate_type": null, "audience_id": "125451003", "name": "Airbyte2"}, "emitted_at": 1697723597280}
+{"stream": "audiences", "data": {"is_valid": false, "audience_type": "Website Audience", "expired_time": "2022-10-20 07:10:04", "create_time": "2021-10-20 07:10:04", "is_expiring": true, "cover_num": 0, "shared": false, "is_creator": true, "calculate_type": null, "audience_id": "125450951", "name": "Airbyte"}, "emitted_at": 1697723597284}
+{"stream": "creative_assets_images", "data": {"material_id": "7080121373767221250", "signature": "2fb70f79f5b11f1bf102039bdd9315df", "image_url": "https://p19-ad-site-sign-sg.ibyteimg.com/v0201/7f371ff6f0764f8b8ef4f37d7b980d50~tplv-d5opwmad15-image.jpeg?x-expires=2013083646&x-signature=40Idxl%2BrjckMLC6YGb32p%2FK3v3g%3D", "is_carousel_usable": false, "size": 33342, "displayable": false, "image_id": "v0201/7f371ff6f0764f8b8ef4f37d7b980d50", "width": 720, "format": "jpeg", "create_time": "2022-03-28T12:11:34Z", "file_name": "7080121373767221250", "height": 1280, "modify_time": "2022-03-28T12:09:10Z"}, "emitted_at": 1697723646940}
+{"stream": "creative_assets_images", "data": {"material_id": "7080116242086625281", "signature": "f564bd399410d9c271f79f0c785414e1", "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "is_carousel_usable": true, "size": 117823, "displayable": false, "image_id": "ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "width": 720, "format": "jpeg", "create_time": "2022-03-28T11:50:49Z", "file_name": "a2_1648468243469.png", "height": 1280, "modify_time": "2022-03-28T11:50:50Z"}, "emitted_at": 1697723646943}
+{"stream": "creative_assets_images", "data": {"material_id": "7080114557281419265", "signature": "68fa13980c042ab6f998bf771375ae15", "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/31e1ea531418f2526783eebea4d43ae3", "is_carousel_usable": true, "size": 216662, "displayable": false, "image_id": "ad-site-i18n-sg/31e1ea531418f2526783eebea4d43ae3", "width": 1280, "format": "jpeg", "create_time": "2022-03-28T11:42:28Z", "file_name": "air.png", "height": 720, "modify_time": "2022-03-28T11:42:29Z"}, "emitted_at": 1697723646945}
+{"stream": "creative_assets_videos", "data": {"allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkpnbc77ucmin3t88g", "height": 1280, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/oEnxmUxanQZbiaZzA9eAQfNdQBb1lzBIDRgVDL~tplv-noop.image?x-expires=1697745278&x-signature=AsbW5pdXMdXYFA3TJ3ZZ9hVFeR0%3D", "modify_time": "2023-10-16T14:46:55Z", "preview_url_expire_time": "2023-10-19 19:54:38", "format": "mp4", "displayable": true, "bit_rate": 3640400, "signature": "62a98cced4a71e2fe6e1edb77ca9dfb7", "material_id": "7290567877181833218", "preview_url": "http://v16m-default.akamaized.net/529e958f65b2c3924fcea48487ab8653/6531897e/video/tos/alisg/tos-alisg-ve-0051c001-sg/oUQJeNBlILamze9UZixnqggDQQBbAdaGxIGVDU/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1666&bt=833&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TqgOpZiyq8Z&mime_type=video_mp4&qs=0&rc=ZGc8ZmQ7Omg2Ozs7ZzhpZ0Bpajs2bGY6ZnFubjMzODYzNEAxYS8wLmMyNjQxMy0uNS8uYSM1cWtqcjRfbWpgLS1kMC1zcw%3D%3D&l=2023101913540719107ED36B1B69182C5C&btag=e00088000", "size": 13657417, "file_name": "Video16974675946002_Heartwarming Atmosphere Pops with Piano Main(827850)", "duration": 30.013, "create_time": "2023-10-16T14:46:55Z", "width": 720, "allow_download": true}, "emitted_at": 1697723649195}
+{"stream": "creative_assets_videos", "data": {"allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkpmbc77u1rm8g4jg0", "height": 1280, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/ogy2o5t3sATEqAfmBACIZNrAWlBEwWt8SZTzhg~tplv-noop.image?x-expires=1697745278&x-signature=U9UXLX%2FGpFVaXXDfmbYKJ%2FRw2mU%3D", "modify_time": "2023-10-16T14:46:54Z", "preview_url_expire_time": "2023-10-19 19:54:38", "format": "mp4", "displayable": true, "bit_rate": 3762666, "signature": "45aa9a03c4b7485e7b98251de9c1e48b", "material_id": "7290567861675278338", "preview_url": "http://v16m-default.akamaized.net/e3ddc851375bfc1da3eee62846f3e7ca/6531897e/video/tos/alisg/tos-alisg-ve-0051c001-sg/o4tZwlzQdEyGfICZsotAm3sOQBTlTqBEAhAgYI/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&br=3524&bt=1762&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TqgOpZiyq8Z&mime_type=video_mp4&qs=0&rc=aGY5PDxnO2U6ZDQ6NzYzaEBpM21qcDQ6ZnBubjMzODYzNEAvYzYtXjMtNjQxMmJgNGBfYSNkMTVvcjRfbWpgLS1kMC1zcw%3D%3D&l=2023101913540719107ED36B1B69182C5C&btag=e00088000", "size": 14116114, "file_name": "Video16974675945900_Whale, sea, electronica(859574)", "duration": 30.013, "create_time": "2023-10-16T14:46:54Z", "width": 720, "allow_download": true}, "emitted_at": 1697723649198}
+{"stream": "creative_assets_videos", "data": {"allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkpm3c77u7v9shhf8g", "height": 1280, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/ogelkgzkaAEPANUZgEeRLTQnYbnvYNUDBDB0tI~tplv-noop.image?x-expires=1697745278&x-signature=mdCZS0HP1tNoMIXWO6MSyKjlbgI%3D", "modify_time": "2023-10-16T14:46:50Z", "preview_url_expire_time": "2023-10-19 19:54:38", "format": "mp4", "displayable": true, "bit_rate": 3671087, "signature": "a492cfd6c89ba8312b159552b3a3c792", "material_id": "7290567856034070530", "preview_url": "http://v16m-default.akamaized.net/503be8b0ee76d4dec055e84343d31b0c/6531897e/video/tos/alisg/tos-alisg-ve-0051c001-sg/oUFNvZDEIBglY8LbnTeQtkoYRPzkQDAenBUNQP/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&br=2846&bt=1423&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TqgOpZiyq8Z&mime_type=video_mp4&qs=0&rc=aTMzNjZlaGdpOzZkZGlnN0BpamlrPDo6ZnBubjMzODYzNEA0XjBjYl5eNWAxM181M15jYSM1ZXBzcjQwbWpgLS1kMC1zcw%3D%3D&l=2023101913540719107ED36B1B69182C5C&btag=e00088000", "size": 13772545, "file_name": "Video16974675946053_U MAKE ME GO", "duration": 30.013, "create_time": "2023-10-16T14:46:50Z", "width": 720, "allow_download": true}, "emitted_at": 1697723649201}