🐛 Source Facebook marketing: reduce start date by one day (#27483)

* Connector health: source hubspot, gitlab, snapchat-marketing: fix builds * #1975 source fb marketing: reduce start date by one day * source fb marketing: upd changelog * 🤖 Auto format source-facebook-marketing code [skip ci] * format code * #1975 source fb marketing: remove cloud version override * source facebook marketing: update CAT config --------- Co-authored-by: octavia-squidington-iii <octavia-squidington-iii@users.noreply.github.com> Co-authored-by: Augustin <augustin@airbyte.io>
airbytehq · Jun 21, 2023 · 076065c · 076065c
1 parent aaf8f34
commit 076065c
Show file tree

Hide file tree

Showing 11 changed files with 46 additions and 24 deletions.
diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile
@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.4.2
+LABEL io.airbyte.version=0.4.3
 LABEL io.airbyte.name=airbyte/source-facebook-marketing
diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml
@@ -6,8 +6,6 @@ acceptance_tests:
   spec:
     tests:
       - spec_path: "integration_tests/spec.json"
-        backward_compatibility_tests_config:
-          disable_for_version: 0.3.6 # pattern for Account ID was added
   connection:
     tests:
       - config_path: "secrets/config.json"
@@ -86,3 +84,7 @@ acceptance_tests:
     tests:
       - config_path: "secrets/config.json"
         timeout_seconds: 4800
+        ignored_fields:
+          activities:
+            - name: extra_data
+              bypass_reason: "image_url in extra_data is changing frequently"
diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py
@@ -1,3 +1,3 @@
 #
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
-  dockerImageTag: 0.4.2
+  dockerImageTag: 0.4.3
   dockerRepository: airbyte/source-facebook-marketing
   githubIssueLabel: source-facebook-marketing
   icon: facebook.svg
@@ -14,7 +14,6 @@ data:
   registries:
     cloud:
       enabled: true
-      dockerImageTag: 0.4.2 
     oss:
       enabled: true
   releaseStage: generally_available

diff --git a/...e-integrations/connectors/source-facebook-marketing/source_facebook_marketing/__init__.py b/...e-integrations/connectors/source-facebook-marketing/source_facebook_marketing/__init__.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 from facebook_business import api
 

diff --git a/...tions/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py b/...tions/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py
@@ -19,6 +19,8 @@
 from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch, FacebookBadObjectError, FacebookResponse
 from source_facebook_marketing.streams.common import retry_pattern
 
+from ..utils import validate_start_date
+
 logger = logging.getLogger("airbyte")
 
 
@@ -189,7 +191,6 @@ class InsightAsyncJob(AsyncJob):
 
     job_timeout = pendulum.duration(hours=1)
     page_size = 100
-    INSIGHTS_RETENTION_PERIOD = pendulum.duration(months=37)
 
     def __init__(self, edge_object: Union[AdAccount, Campaign, AdSet, Ad], params: Mapping[str, Any], **kwargs):
         """Initialize
@@ -242,10 +243,9 @@ def _split_by_edge_class(self, edge_class: Union[Type[Campaign], Type[AdSet], Ty
         params = dict(copy.deepcopy(self._params))
         # get objects from attribution window as well (28 day + 1 current day)
         new_start = self._interval.start - pendulum.duration(days=28 + 1)
-        oldest_date = pendulum.today().date() - self.INSIGHTS_RETENTION_PERIOD
-        new_start = max(new_start, oldest_date)
-        params.update(fields=[pk_name], level=level)
+        new_start = validate_start_date(new_start)
         params["time_range"].update(since=new_start.to_date_string())
+        params.update(fields=[pk_name], level=level)
         params.pop("time_increment")  # query all days
         result = self._edge_object.get_insights(params=params)
         ids = set(row[pk_name] for row in result)

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py
@@ -3,9 +3,10 @@
 #
 
 import logging
+from typing import Union
 
 import pendulum
-from pendulum import DateTime
+from pendulum import Date, DateTime
 
 logger = logging.getLogger("airbyte")
 
@@ -14,32 +15,45 @@
 # HTTP response.
 # https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights/#overview
 DATA_RETENTION_PERIOD = 37
+DateOrDateTime = Union[Date, DateTime]
 
 
-def validate_start_date(start_date: DateTime) -> DateTime:
-    now = pendulum.now(tz=start_date.tzinfo)
-    today = now.replace(microsecond=0, second=0, minute=0, hour=0)
+def cast_to_type(input_date: DateOrDateTime, target_date: DateOrDateTime) -> DateOrDateTime:
+    # casts `target_date` to the type of `input_date`
+    if type(target_date) == type(input_date):
+        return target_date
+    if isinstance(target_date, DateTime):
+        return target_date.date()
+    return pendulum.datetime(target_date.year, target_date.month, target_date.day)
+
+
+def validate_start_date(start_date: DateOrDateTime) -> DateOrDateTime:
+    today = cast_to_type(start_date, pendulum.today())
     retention_date = today.subtract(months=DATA_RETENTION_PERIOD)
     if retention_date.day != today.day:
         # `.subtract(months=37)` can be erroneous, for instance:
         # 2023-03-31 - 37 month = 2020-02-29 which is incorrect, should be 2020-03-01
         # that's why we're adjusting the date to the 1st day of the next month
         retention_date = retention_date.replace(month=retention_date.month + 1, day=1)
 
-    if start_date > now:
+    # FB does not provide precise description of how we should calculate the 37 months datetime difference.
+    # As timezone difference can result in an additional time delta, this is a reassurance we stay inside the 37 months limit.
+    retention_date = retention_date.add(days=1)
+
+    if start_date > today:
         message = f"The start date cannot be in the future. Set start date to today's date - {today}."
         logger.warning(message)
-        return today
+        return cast_to_type(start_date, today)
     elif start_date < retention_date:
         message = (
             f"The start date cannot be beyond {DATA_RETENTION_PERIOD} months from the current date. Set start date to {retention_date}."
         )
         logger.warning(message)
-        return retention_date
+        return cast_to_type(start_date, retention_date)
     return start_date
 
 
-def validate_end_date(start_date: DateTime, end_date: DateTime) -> DateTime:
+def validate_end_date(start_date: DateOrDateTime, end_date: DateOrDateTime) -> DateOrDateTime:
     if start_date > end_date:
         message = f"The end date must be after start date. Set end date to {start_date}."
         logger.warning(message)

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/__init__.py
@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py
@@ -342,7 +342,10 @@ def test_split_job(self, mocker, api, edge_class, next_edge_class, id_field):
                 "breakdowns": [],
                 "fields": [id_field],
                 "level": next_edge_class.__name__.lower(),
-                "time_range": {"since": (today - pendulum.duration(months=37)).to_date_string(), "until": end.to_date_string()},
+                "time_range": {
+                    "since": (today - pendulum.duration(months=37) + pendulum.duration(days=1)).to_date_string(),
+                    "until": end.to_date_string()
+                },
             }
         )
         assert len(small_jobs) == 3

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py
@@ -22,10 +22,10 @@
         (
             "start_date",
             pendulum.local(2019, 1, 1),
-            pendulum.local(2020, 3, 1),
+            pendulum.local(2020, 3, 2),
             [
                 f"The start date cannot be beyond 37 months from the current date. "
-                f"Set start date to {pendulum.local(2020, 3, 1)}."
+                f"Set start date to {pendulum.local(2020, 3, 2)}."
             ]
         ),
         (

diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md
@@ -165,8 +165,9 @@ Please be informed that the connector uses the `lookback_window` parameter to pe
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 0.4.2   | 2023-06-09 | [27201](https://github.com/airbytehq/airbyte/pull/27201) | Add `complete_oauth_server_output_specification` to spec                   |
-| 0.4.1   | 2023-06-02 | [26941](https://github.com/airbytehq/airbyte/pull/26941) | Remove `authSpecification` from spec.json, use `advanced_auth` instead                                                                                                                                                                                                                                                                  |
+| 0.4.3   | 2023-05-12 | [27483](https://github.com/airbytehq/airbyte/pull/27483) | Reduce replication start date by one more day                                                                                                                                                                                                                                                     |
+| 0.4.2   | 2023-06-09 | [27201](https://github.com/airbytehq/airbyte/pull/27201) | Add `complete_oauth_server_output_specification` to spec                                                                                                                                                                                                                                          |
+| 0.4.1   | 2023-06-02 | [26941](https://github.com/airbytehq/airbyte/pull/26941) | Remove `authSpecification` from spec.json, use `advanced_auth` instead                                                                                                                                                                                                                            |
 | 0.4.0   | 2023-05-29 | [26720](https://github.com/airbytehq/airbyte/pull/26720) | Add Prebuilt Ad Insights reports                                                                                                                                                                                                                                                                  |
 | 0.3.7   | 2023-05-12 | [26000](https://github.com/airbytehq/airbyte/pull/26000) | Handle config errors                                                                                                                                                                                                                                                                              |
 | 0.3.6   | 2023-04-27 | [22999](https://github.com/airbytehq/airbyte/pull/22999) | Specified date formatting in specification                                                                                                                                                                                                                                                        |