Source Tiktok Marketing: add new fields to ad reports streams

airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile

@@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=3.3.0
+LABEL io.airbyte.version=3.3.1
 LABEL io.airbyte.name=airbyte/source-tiktok-marketing

airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml

@@ -6,7 +6,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
-  dockerImageTag: 3.3.0
+  dockerImageTag: 3.3.1
   dockerRepository: airbyte/source-tiktok-marketing
   githubIssueLabel: source-tiktok-marketing
   icon: tiktok.svg

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json

@@ -140,6 +140,39 @@
         },
         "frequency": {
           "type": ["null", "string"]
+        },
+        "total_purchase_value": {
+          "type": ["null", "string"]
+        },
+        "total_onsite_shopping_value": {
+          "type": ["null", "string"]
+        },
+        "onsite_shopping": {
+          "type": ["null", "string"]
+        },
+        "vta_purchase": {
+          "type": ["null", "string"]
+        },
+        "cta_purchase": {
+          "type": ["null", "string"]
+        },
+        "cta_conversion": {
+          "type": ["null", "string"]
+        },
+        "vta_conversion": {
+          "type": ["null", "string"]
+        },
+        "total_pageview": {
+          "type": ["null", "string"]
+        },
+        "complete_payment": {
+          "type": ["null", "string"]
+        },
+        "value_per_complete_payment": {
+          "type": ["null", "string"]
+        },
+        "total_complete_payment_rate": {
+          "type": ["null", "string"]
         }
       }
     },

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py

@@ -51,6 +51,17 @@
     "real_time_app_install",
     "real_time_app_install_cost",
     "app_install",
+    "total_purchase_value",
+    "total_onsite_shopping_value",
+    "onsite_shopping",
+    "vta_purchase",
+    "cta_purchase",
+    "vta_conversion",
+    "cta_conversion",
+    "total_pageview",
+    "complete_payment",
+    "value_per_complete_payment",
+    "total_complete_payment_rate",
 ]
 
 T = TypeVar("T")
@@ -656,7 +667,24 @@ def _get_metrics(self):
             )
 
         if self.report_level == ReportLevel.AD:
-            result.extend(["adgroup_id", "ad_name", "ad_text"])
+            result.extend(
+                [
+                    "adgroup_id",
+                    "ad_name",
+                    "ad_text",
+                    "total_purchase_value",
+                    "total_onsite_shopping_value",
+                    "onsite_shopping",
+                    "vta_purchase",
+                    "vta_conversion",
+                    "cta_purchase",
+                    "cta_conversion",
+                    "total_pageview",
+                    "complete_payment",
+                    "value_per_complete_payment",
+                    "total_complete_payment_rate",
+                ]
+            )
 
         return result
 

airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py

@@ -68,13 +68,15 @@ def advertiser_ids_fixture():
     ],
 )
 def test_get_time_interval(pendulum_now_mock, granularity, intervals_len):
-    intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity)
+    intervals = BasicReports._get_time_interval(
+        start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity)
     assert len(list(intervals)) == intervals_len
 
 
 @patch.object(pendulum, "now", return_value=pendulum.parse("2018-12-25"))
 def test_get_time_interval_past(pendulum_now_mock_past):
-    intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY)
+    intervals = BasicReports._get_time_interval(
+        start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY)
     assert len(list(intervals)) == 1
 
 
@@ -109,25 +111,32 @@ def test_stream_slices_basic_sandbox(advertiser_ids, config_name, slices_expecte
         (
             Daily,
             [
-                {"advertiser_id": 1, "end_date": "2020-01-30", "start_date": "2020-01-01"},
-                {"advertiser_id": 1, "end_date": "2020-02-29", "start_date": "2020-01-31"},
-                {"advertiser_id": 1, "end_date": "2020-03-01", "start_date": "2020-03-01"},
-                {"advertiser_id": 2, "end_date": "2020-01-30", "start_date": "2020-01-01"},
-                {"advertiser_id": 2, "end_date": "2020-02-29", "start_date": "2020-01-31"},
-                {"advertiser_id": 2, "end_date": "2020-03-01", "start_date": "2020-03-01"},
+                {"advertiser_id": 1, "end_date": "2020-01-30",
+                    "start_date": "2020-01-01"},
+                {"advertiser_id": 1, "end_date": "2020-02-29",
+                    "start_date": "2020-01-31"},
+                {"advertiser_id": 1, "end_date": "2020-03-01",
+                    "start_date": "2020-03-01"},
+                {"advertiser_id": 2, "end_date": "2020-01-30",
+                    "start_date": "2020-01-01"},
+                {"advertiser_id": 2, "end_date": "2020-02-29",
+                    "start_date": "2020-01-31"},
+                {"advertiser_id": 2, "end_date": "2020-03-01",
+                    "start_date": "2020-03-01"},
             ],
         ),
     ],
 )
 def test_stream_slices_report(advertiser_ids, granularity, slices_expected, pendulum_now_mock):
-    slices = get_report_stream(AdsReports, granularity)(**CONFIG).stream_slices()
+    slices = get_report_stream(AdsReports, granularity)(
+        **CONFIG).stream_slices()
     assert list(slices) == slices_expected
 
 
 @pytest.mark.parametrize(
     "stream, metrics_number",
     [
-        (AdsReports, 54),
+        (AdsReports, 65),
         (AdGroupsReports, 51),
         (AdvertisersReports, 29),
         (CampaignsReports, 28),
@@ -143,7 +152,7 @@ def test_basic_reports_get_metrics_day(stream, metrics_number):
 @pytest.mark.parametrize(
     "stream, metrics_number",
     [
-        (AdsReports, 54),
+        (AdsReports, 65),
         (AdGroupsReports, 51),
         (AdvertisersReports, 27),
         (CampaignsReports, 28),
@@ -166,7 +175,8 @@ def test_basic_reports_get_metrics_lifetime(stream, metrics_number):
     ],
 )
 def test_basic_reports_get_reporting_dimensions_lifetime(stream, dimensions_expected):
-    dimensions = get_report_stream(stream, Lifetime)(**CONFIG)._get_reporting_dimensions()
+    dimensions = get_report_stream(stream, Lifetime)(
+        **CONFIG)._get_reporting_dimensions()
     assert dimensions == dimensions_expected
 
 
@@ -177,11 +187,13 @@ def test_basic_reports_get_reporting_dimensions_lifetime(stream, dimensions_expe
         (AdGroupsReports, ["adgroup_id", "stat_time_day"]),
         (AdvertisersReports, ["advertiser_id", "stat_time_day"]),
         (CampaignsReports, ["campaign_id", "stat_time_day"]),
-        (AdvertisersAudienceReports, ["advertiser_id", "stat_time_day", "gender", "age"]),
+        (AdvertisersAudienceReports, [
+         "advertiser_id", "stat_time_day", "gender", "age"]),
     ],
 )
 def test_basic_reports_get_reporting_dimensions_day(stream, dimensions_expected):
-    dimensions = get_report_stream(stream, Daily)(**CONFIG)._get_reporting_dimensions()
+    dimensions = get_report_stream(stream, Daily)(
+        **CONFIG)._get_reporting_dimensions()
     assert dimensions == dimensions_expected
 
 
@@ -200,8 +212,10 @@ def test_basic_reports_cursor_field(granularity, cursor_field_expected):
 
 
 def test_request_params():
-    stream_slice = {"advertiser_id": 1, "start_date": "2020", "end_date": "2021"}
-    params = get_report_stream(AdvertisersAudienceReports, Daily)(**CONFIG).request_params(stream_slice=stream_slice)
+    stream_slice = {"advertiser_id": 1,
+                    "start_date": "2020", "end_date": "2021"}
+    params = get_report_stream(AdvertisersAudienceReports, Daily)(
+        **CONFIG).request_params(stream_slice=stream_slice)
     assert params == {
         "advertiser_id": 1,
         "data_level": "AUCTION_ADVERTISER",
@@ -226,11 +240,14 @@ def test_get_updated_state():
         # state should be empty while stream is reading records
         ads.max_cursor_date = "2020-01-08 00:00:00"
         is_finished.return_value = False
-        state1 = ads.get_updated_state(current_stream_state=state, latest_record={})
+        state1 = ads.get_updated_state(
+            current_stream_state=state, latest_record={})
         assert state1 == {"modify_time": ""}
 
         # state should be updated only when all records have been read (is_finished = True)
         is_finished.return_value = True
-        state2 = ads.get_updated_state(current_stream_state=state, latest_record={})
-        state2_modify_time = state2["modify_time"]  # state2_modify_time is JsonUpdatedState object
+        state2 = ads.get_updated_state(
+            current_stream_state=state, latest_record={})
+        # state2_modify_time is JsonUpdatedState object
+        state2_modify_time = state2["modify_time"]
         assert state2_modify_time.dict() == "2020-01-08 00:00:00"

airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py

@@ -81,7 +81,7 @@ def unixtime2str(unix_time: int) -> str:
 def test_random_items(prepared_prod_args):
     stream = Ads(**prepared_prod_args)
     advertiser_count = 100
-    test_advertiser_ids = set([random_integer() for _ in range(advertiser_count)])
+    test_advertiser_ids = set([str(random_integer()) for _ in range(advertiser_count)])
     advertiser_count = len(test_advertiser_ids)
     page_size = 100
     with requests_mock.Mocker() as m: