Source Facebook Marketing: automatically reduce batch size (#29610)

Co-authored-by: Alexandre Girard <alexandre@airbyte.io>
airbytehq · Aug 19, 2023 · ef0a3ef · ef0a3ef
1 parent 4ac3322
commit ef0a3ef
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 4 deletions.
diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile
@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=1.1.4
+LABEL io.airbyte.version=1.1.5
 LABEL io.airbyte.name=airbyte/source-facebook-marketing
diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json
@@ -374,6 +374,7 @@
                 "default": 50,
                 "order": 9,
                 "exclusiveMinimum": 0,
+                "maximum": 50,
                 "type": "integer"
             },
             "action_breakdowns_allow_empty": {

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
-  dockerImageTag: 1.1.4
+  dockerImageTag: 1.1.5
   dockerRepository: airbyte/source-facebook-marketing
   githubIssueLabel: source-facebook-marketing
   icon: facebook.svg

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py
@@ -201,14 +201,16 @@ class Config:
         default=28,
     )
 
-    max_batch_size: Optional[PositiveInt] = Field(
+    max_batch_size: Optional[int] = Field(
         title="Maximum size of Batched Requests",
         order=9,
         description=(
             "Maximum batch size used when sending batch requests to Facebook API. "
             "Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases."
         ),
         default=50,
+        gt=0,
+        le=50,
     )
 
     action_breakdowns_allow_empty: bool = Field(

diff --git a/...ns/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/...ns/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py
@@ -78,9 +78,16 @@ def failure(response: FacebookResponse, request: Optional[FacebookRequest] = Non
             # although it is Optional in the signature for compatibility, we need it always
             assert request, "Missing a request object"
             resp_body = response.json()
-            if not isinstance(resp_body, dict) or resp_body.get("error", {}).get("code") != FACEBOOK_BATCH_ERROR_CODE:
+            if not isinstance(resp_body, dict) or (
+                resp_body.get("error", {}).get("code") != FACEBOOK_BATCH_ERROR_CODE
+                and resp_body.get("error", {}).get("message")
+                != "Please reduce the amount of data you're asking for, then retry your request"
+            ):
                 # response body is not a json object or the error code is different
                 raise RuntimeError(f"Batch request failed with response: {resp_body}")
+            if resp_body.get("error", {}).get("message") == "Please reduce the amount of data you're asking for, then retry your request":
+                logger.warning("Caught retryable error: Too much data was requested in batch. Reducing batch size...")
+                self.max_batch_size = int(self.max_batch_size / 2)
             requests_q.put(request)
 
         api_batch: FacebookAdsApiBatch = self._api.api.new_batch()

diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md
@@ -175,6 +175,7 @@ The Facebook Marketing connector uses the `lookback_window` parameter to repeate
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.1.5   | 2023-08-18 | [29610](https://github.com/airbytehq/airbyte/pull/29610) | Automatically reduce batch size                                                                                                                                                                                                                                                                   |
 | 1.1.4   | 2023-08-08 | [29412](https://github.com/airbytehq/airbyte/pull/29412) | Add new custom_audience stream                                                                                                                                                                                                                                                                    |
 | 1.1.3   | 2023-08-08 | [29208](https://github.com/airbytehq/airbyte/pull/29208) | Add account type validation during check                                                                                                                                                                                                                                                          |
 | 1.1.2   | 2023-08-03 | [29042](https://github.com/airbytehq/airbyte/pull/29042) | Fix broken `advancedAuth` references for `spec`                                                                                                                                                                                                                                                   |