Source tiktok: chunk advertiser IDs (#22309)

* #1141 source tiktok: chunk advertiser IDs

* #1141 source tiktok: upd changelog

airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile

@@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=2.0.1
+LABEL io.airbyte.version=2.0.2
 LABEL io.airbyte.name=airbyte/source-tiktok-marketing

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py

@@ -88,7 +88,9 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
         Tests if the input configuration can be used to successfully connect to the integration
         """
         try:
-            next(Advertisers(**self._prepare_stream_args(config)).read_records(SyncMode.full_refresh))
+            advertisers = Advertisers(**self._prepare_stream_args(config))
+            for slice_ in advertisers.stream_slices():
+                next(advertisers.read_records(SyncMode.full_refresh, stream_slice=slice_))
         except Exception as err:
             return False, err
         return True, None

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py

@@ -278,7 +278,7 @@ def __init__(self, start_date: str, end_date: str, **kwargs):
     def convert_array_param(arr: List[Union[str, int]]) -> str:
         return json.dumps(arr)
 
-    def get_advertiser_ids(self) -> Iterable[int]:
+    def get_advertiser_ids(self) -> List[int]:
         if self.is_sandbox:
             # for sandbox: just return advertiser_id provided in spec
             ids = [self._advertiser_id]
@@ -306,9 +306,8 @@ def is_finished(self):
     def request_params(
         self,
         stream_state: Mapping[str, Any] = None,
-        next_page_token: Mapping[str, Any] = None,
         stream_slice: Mapping[str, Any] = None,
-        **kwargs,
+        next_page_token: Mapping[str, Any] = None,
     ) -> MutableMapping[str, Any]:
         params = {"page_size": self.page_size}
         if self.fields:
@@ -412,17 +411,23 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late
 class Advertisers(FullRefreshTiktokStream):
     """Docs: https://ads.tiktok.com/marketing_api/docs?id=1708503202263042"""
 
-    def request_params(self, **kwargs) -> MutableMapping[str, Any]:
-        params = super().request_params(**kwargs)
-        params["advertiser_ids"] = self.convert_array_param(self.get_advertiser_ids())
-        return params
+    def request_params(
+        self,
+        stream_state: Mapping[str, Any] = None,
+        stream_slice: Mapping[str, Any] = None,
+        next_page_token: Mapping[str, Any] = None,
+    ) -> MutableMapping[str, Any]:
+        stream_slice = stream_slice or {}
+        return {key: self.convert_array_param(value) for key, value in stream_slice.items()}
 
     def path(self, *args, **kwargs) -> str:
         return "advertiser/info/"
 
     def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
-        """this stream must work with the default slice logic"""
-        yield None
+        ids = self.get_advertiser_ids()
+        start, end, step = 0, len(ids), 100
+        for i in range(start, end, step):
+            yield {"advertiser_ids": ids[i: min(end, i + step)]}
 
 
 class Campaigns(IncrementalTiktokStream):

airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py

@@ -78,9 +78,10 @@ def test_get_time_interval_past(pendulum_now_mock_past):
     assert len(list(intervals)) == 1
 
 
+@patch("source_tiktok_marketing.streams.AdvertiserIds.read_records", MagicMock(return_value=[{"advertiser_id": i} for i in range(354)]))
 def test_stream_slices_advertisers():
     slices = Advertisers(**CONFIG).stream_slices()
-    assert list(slices) == [None]
+    assert len(list(slices)) == 4  # math.ceil(354 / 100)
 
 
 @pytest.mark.parametrize(

airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py

@@ -167,8 +167,9 @@ def logger_mock_fixture():
 
 
 def test_source_check_connection_ok(config, logger_mock):
-    with patch.object(Advertisers, "read_records", return_value=iter([1])):
-        assert SourceTiktokMarketing().check_connection(logger_mock, config=config) == (True, None)
+    with patch.object(Advertisers, "stream_slices"):
+        with patch.object(Advertisers, "read_records", return_value=iter([1])):
+            assert SourceTiktokMarketing().check_connection(logger_mock, config=config) == (True, None)
 
 
 def test_source_check_connection_failed(config, logger_mock):

docs/integrations/sources/tiktok-marketing.md

@@ -550,7 +550,8 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark
 
 | Version | Date       | Pull Request                                             | Subject                                                                                       |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------|
-| 2.0.1   | 2023-01-27 | [22044](https://github.com/airbytehq/airbyte/pull/22044) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                     |
+| 2.0.2   | 2023-02-02 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Chunk Advertiser IDs                                                                          |
+| 2.0.1   | 2023-01-27 | [22044](https://github.com/airbytehq/airbyte/pull/22044) | Set `AvailabilityStrategy` for streams explicitly to `None`                                   |
 | 2.0.0   | 2022-12-20 | [20415](https://github.com/airbytehq/airbyte/pull/20415) | Update schema types for `AudienceReports` and `BasicReports` streams.                         |
 | 1.0.1   | 2022-12-16 | [20598](https://github.com/airbytehq/airbyte/pull/20598) | Remove Audience Reports with Hourly granularity due to deprecated dimension.                  |
 | 1.0.0   | 2022-12-05 | [19758](https://github.com/airbytehq/airbyte/pull/19758) | Convert `mobile_app_id` from integer to string in AudienceReport streams.                     |