Source Facebook Marketing: fix start_date selection behaviour

airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml

@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
-  dockerImageTag: 2.0.1
+  dockerImageTag: 2.0.2
   dockerRepository: airbyte/source-facebook-marketing
   documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing
   githubIssueLabel: source-facebook-marketing
@@ -30,7 +30,7 @@ data:
   releases:
     breakingChanges:
       2.0.0:
-        message: "All Ads-Insights-* streams now have updated schemas. Users will need to retest source confguration, refresh the source schema and reset affected streams after upgrading. For more information [visit](https://docs.airbyte.com/integrations/sources/facebook-marketing-migrations)"
+        message: "All Ads-Insights-* streams now have updated schemas. Users will need to retest source configuration, refresh the source schema and reset affected streams after upgrading. Please pay attention that data older than 37 months will become unavailable due to FaceBook limitations. For more information [visit](https://docs.airbyte.com/integrations/sources/facebook-marketing-migrations)"
         upgradeDeadline: "2024-03-17"
         scopedImpact:
           - scopeType: stream

airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.0.1"
+version = "2.0.2"
 name = "source-facebook-marketing"
 description = "Source implementation for Facebook Marketing."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py

@@ -314,6 +314,10 @@ def _get_start_date(self) -> Mapping[str, pendulum.Date]:
                 start_date = min(start_date, refresh_date)
             else:
                 start_date = self._start_date
+
+            if start_date < self._start_date:
+                logger.warning(f"Ignore provided state and start sync from start_date ({self._start_date}).")
+            start_date = max(start_date, self._start_date)
             if start_date < oldest_date:
                 logger.warning(
                     f"Loading insights older then {self.INSIGHTS_RETENTION_PERIOD} is not possible. Start sync from {oldest_date}."

airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py

@@ -334,9 +334,9 @@ def test_stream_slices_with_state(self, api, async_manager_mock, start_date, som
         args, kwargs = async_manager_mock.call_args
         generated_jobs = list(kwargs["jobs"])
         # assert that we sync all periods including insight_lookback_period
-        assert len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1
-        assert generated_jobs[0].interval.start == cursor_value.date() - stream.insights_lookback_period
-        assert generated_jobs[1].interval.start == cursor_value.date() - stream.insights_lookback_period + duration(days=1)
+        assert len(generated_jobs) == (end_date.date() - start_date).days + 1
+        assert generated_jobs[0].interval.start == start_date.date()
+        assert generated_jobs[1].interval.start == start_date.date() + duration(days=1)
 
     def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, recent_start_date, some_config):
         """Stream will use start_date when close to now and start_date close to now"""
@@ -363,15 +363,15 @@ def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, re
         async_manager_mock.assert_called_once()
         args, kwargs = async_manager_mock.call_args
         generated_jobs = list(kwargs["jobs"])
-        assert len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1
-        assert generated_jobs[0].interval.start == cursor_value.date() - stream.insights_lookback_period
-        assert generated_jobs[1].interval.start == cursor_value.date() - stream.insights_lookback_period + duration(days=1)
+        assert len(generated_jobs) == (end_date.date() - start_date).days + 1
+        assert generated_jobs[0].interval.start == start_date.date()
+        assert generated_jobs[1].interval.start == start_date.date() + duration(days=1)
 
     @pytest.mark.parametrize("state_format", ["old_format", "new_format"])
     def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, start_date, some_config, state_format):
         """Stream will use cursor_value from state, but will skip saved slices"""
-        end_date = start_date + duration(days=10)
-        cursor_value = start_date + duration(days=5)
+        end_date = start_date + duration(days=40)
+        cursor_value = start_date + duration(days=32)
 
         if state_format == "old_format":
             state = {
@@ -410,7 +410,7 @@ def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, star
         async_manager_mock.assert_called_once()
         args, kwargs = async_manager_mock.call_args
         generated_jobs = list(kwargs["jobs"])
-        assert len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1, "should be 34 slices because we ignore slices which are within insights_lookback_period"
+        assert len(generated_jobs) == (end_date.date() - (cursor_value.date() - stream.insights_lookback_period)).days + 1, "should be 37 slices because we ignore slices which are within insights_lookback_period"
         assert generated_jobs[0].interval.start == cursor_value.date() - stream.insights_lookback_period
         assert generated_jobs[1].interval.start == cursor_value.date() - stream.insights_lookback_period + duration(days=1)
 

docs/integrations/sources/facebook-marketing-migrations.md

@@ -4,6 +4,11 @@
 
 Streams Ads-Insights-* streams now have updated schemas.
 
+:::danger
+Please note that data older than 37 months will become unavailable due to Facebook limitations.
+It is recommended to create a backup at the destination before proceeding with migration.
+:::
+
 ### Update Custom Insights Reports (this step can be skipped if you did not define any)
 
 1. Select **Sources** in the main navbar.
@@ -20,18 +25,18 @@ Streams Ads-Insights-* streams now have updated schemas.
     1. Select **Refresh source schema**.
     2. Select **OK**.
 
-```note
+:::note
 Any detected schema changes will be listed for your review.
-```
+:::
 
 3. Select **Save changes** at the bottom of the page.
     1. Ensure the **Reset affected streams** option is checked.
-```note
+:::note
 Depending on destination type you may not be prompted to reset your data.
-```
+:::
 4. Select **Save connection**. 
-```note
+:::note
 This will reset the data in your destination and initiate a fresh sync.
-```
+:::
 
 For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset).

docs/integrations/sources/facebook-marketing.md

@@ -200,6 +200,7 @@ The Facebook Marketing connector uses the `lookback_window` parameter to repeate
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 2.0.2   | 2024-03-13 | [36025](https://github.com/airbytehq/airbyte/pull/36025) | Fix start_date selection behaviour                                                                                                                                                                                                                                                                |
 | 2.0.1   | 2024-03-08 | [35913](https://github.com/airbytehq/airbyte/pull/35913) | Fix lookback window                                                                                                                                                                                                                                                                               |
 | 2.0.0   | 2024-03-01 | [35746](https://github.com/airbytehq/airbyte/pull/35746) | Update API to `v19.0`                                                                                                                                                                                                                                                                             |
 | 1.4.2   | 2024-02-22 | [35539](https://github.com/airbytehq/airbyte/pull/35539) | Add missing config migration from `include_deleted` field                                                                                                                                                                                                                                         |