From d277560c6518aed7c28f2244a0e837c86e099343 Mon Sep 17 00:00:00 2001 From: vladimir-remar Date: Wed, 27 Apr 2022 16:47:01 +0200 Subject: [PATCH 01/18] WIP: add lookback window to insgiths streams --- .../source_facebook_marketing/source.py | 1 + .../source_facebook_marketing/spec.py | 10 ++++++++++ .../streams/base_insight_streams.py | 19 ++++++++++++------- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index d3e079a561cb3..94e5337c899ba 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -64,6 +64,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: api=api, start_date=config.start_date, end_date=config.end_date, + insights_lookback_window=config.insights_lookback_window ) streams = [ AdAccount(api=api), diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 6152b50ab119a..a9a908d9b4475 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -147,3 +147,13 @@ class Config: "A list which contains insights entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns)" ), ) + + insights_lookback_window: Optional[PositiveInt] = Field( + title="Insights Lookback Window", + order=7, + description=( + "The attribution window" + ), + exclusiveMaximum=28, + default=28, + ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 0aa2378760889..63e6f4f86ce40 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -46,9 +46,6 @@ class AdsInsights(FBMarketingIncrementalStream): # HTTP response. # https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights/#overview INSIGHTS_RETENTION_PERIOD = pendulum.duration(months=37) - # Facebook freezes insight data 28 days after it was generated, which means that all data - # from the past 28 days may have changed since we last emitted it, so we retrieve it again. - INSIGHTS_LOOKBACK_PERIOD = pendulum.duration(days=28) action_breakdowns = ALL_ACTION_BREAKDOWNS level = "ad" @@ -64,6 +61,7 @@ def __init__( breakdowns: List[str] = None, action_breakdowns: List[str] = None, time_increment: Optional[int] = None, + insights_lookback_window: int = None, **kwargs, ): super().__init__(**kwargs) @@ -74,6 +72,7 @@ def __init__( self.breakdowns = breakdowns or self.breakdowns self.time_increment = time_increment or self.time_increment self._new_class_name = name + self._insights_lookback_window = insights_lookback_window # state self._cursor_value: Optional[pendulum.Date] = None # latest period that was read @@ -91,6 +90,14 @@ def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: """Build complex PK based on slices and breakdowns""" return ["date_start", "account_id", "ad_id"] + self.breakdowns + @property + def insights_lookback_period(self): + """ + Facebook freezes insight data 28 days after it was generated, which means that all data + from the past 28 days may have changed since we last emitted it, so we retrieve it again. + """ + return pendulum.duration(days=self._insights_lookback_window) + def list_objects(self, params: Mapping[str, Any]) -> Iterable: """Because insights has very different read_records we don't need this method anymore""" @@ -217,13 +224,12 @@ def _get_start_date(self) -> pendulum.Date: """ today = pendulum.today().date() oldest_date = today - self.INSIGHTS_RETENTION_PERIOD - refresh_date = today - self.INSIGHTS_LOOKBACK_PERIOD - + refresh_date = today - self.insights_lookback_period if self._cursor_value: start_date = self._cursor_value + pendulum.duration(days=self.time_increment) if start_date > refresh_date: logger.info( - f"The cursor value within refresh period ({self.INSIGHTS_LOOKBACK_PERIOD}), start sync from {refresh_date} instead." + f"The cursor value within refresh period ({self.insights_lookback_period}), start sync from {refresh_date} instead." ) start_date = min(start_date, refresh_date) @@ -234,7 +240,6 @@ def _get_start_date(self) -> pendulum.Date: start_date = self._start_date if start_date < oldest_date: logger.warning(f"Loading insights older then {self.INSIGHTS_RETENTION_PERIOD} is not possible. Start sync from {oldest_date}.") - return max(oldest_date, start_date) def request_params(self, **kwargs) -> MutableMapping[str, Any]: From 03d5a7235084a68c8df4d36370e36c60a6b09560 Mon Sep 17 00:00:00 2001 From: vladimir-remar Date: Wed, 27 Apr 2022 17:20:01 +0200 Subject: [PATCH 02/18] update: docs in --- .../source_facebook_marketing/streams/base_insight_streams.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 63e6f4f86ce40..d26217b06902e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -95,6 +95,8 @@ def insights_lookback_period(self): """ Facebook freezes insight data 28 days after it was generated, which means that all data from the past 28 days may have changed since we last emitted it, so we retrieve it again. + But in some cases users my have define their own lookback window, thats + why the value for `insights_lookback_window` is set throught config. """ return pendulum.duration(days=self._insights_lookback_window) From 84efdab79acf06ee26656e1b4b3ceee369fdb924 Mon Sep 17 00:00:00 2001 From: "vladimir.remar" Date: Tue, 10 May 2022 13:48:46 +0200 Subject: [PATCH 03/18] update: add insights_lookback_window in custom insights --- .../source_facebook_marketing/source.py | 1 + .../source_facebook_marketing/spec.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 94e5337c899ba..ccf8b8890a40b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -124,6 +124,7 @@ def _update_insights_streams(self, insights: List[InsightConfig], default_args, time_increment=insight.time_increment, start_date=insight.start_date or default_args["start_date"], end_date=insight.end_date or default_args["end_date"], + insights_lookback_window=insight.insights_lookback_window or default_args["insights_lookback_window"], ) insight_stream = AdsInsights(**args) insights_custom_streams.append(insight_stream) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index a9a908d9b4475..6acee06f9d3db 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -77,6 +77,14 @@ class Config: pattern=DATE_TIME_PATTERN, examples=["2017-01-26T00:00:00Z"], ) + insights_lookback_window: Optional[PositiveInt] = Field( + title="Custom Insights Lookback Window", + description=( + "The attribution window" + ), + exclusiveMaximum=28, + default=28, + ) class ConnectorConfig(BaseConfig): From d30b66d406297c34b768b9bddbf2381690d97d00 Mon Sep 17 00:00:00 2001 From: vladimir remar Date: Thu, 19 May 2022 09:14:59 +0200 Subject: [PATCH 04/18] update connector version in dockerfile and update facebook-marketing.md --- .../connectors/source-facebook-marketing/Dockerfile | 2 +- docs/integrations/sources/facebook-marketing.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index f65fec63dc3e8..4f2bc54593430 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.47 +LABEL io.airbyte.version=0.2.48 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 22f50a839e72f..2e498212bdfe4 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -108,6 +108,8 @@ For more information, see the [Facebook Insights API documentation.](https://dev | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.48 | 2022-04-27 | [12402](https://github.com/airbytehq/airbyte/pull/12402) | Add lookback window to insgiths streams | + | 0.2.47 | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions | | 0.2.46 | 2022-04-22 | [12171](https://github.com/airbytehq/airbyte/pull/12171) | Allow configuration of page_size for requests | | 0.2.45 | 2022-05-03 | [12390](https://github.com/airbytehq/airbyte/pull/12390) | Better retry logic for split-up async jobs | From 71f326ae46d2037b670dfbe744cb565f6b01dbd4 Mon Sep 17 00:00:00 2001 From: vladimir remar Date: Fri, 20 May 2022 08:45:27 +0200 Subject: [PATCH 05/18] formatting using blackFormat --- .../source_facebook_marketing/source.py | 5 +---- .../source_facebook_marketing/spec.py | 8 ++------ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index c3027a44213f5..d8279be293136 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -61,10 +61,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: api = API(account_id=config.account_id, access_token=config.access_token) insights_args = dict( - api=api, - start_date=config.start_date, - end_date=config.end_date, - insights_lookback_window=config.insights_lookback_window + api=api, start_date=config.start_date, end_date=config.end_date, insights_lookback_window=config.insights_lookback_window ) streams = [ AdAccount(api=api), diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 5db54466ab44b..d6b363cd1439d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -79,9 +79,7 @@ class Config: ) insights_lookback_window: Optional[PositiveInt] = Field( title="Custom Insights Lookback Window", - description=( - "The attribution window" - ), + description=("The attribution window"), exclusiveMaximum=28, default=28, ) @@ -168,9 +166,7 @@ class Config: insights_lookback_window: Optional[PositiveInt] = Field( title="Insights Lookback Window", order=7, - description=( - "The attribution window" - ), + description=("The attribution window"), exclusiveMaximum=28, default=28, ) From b1481dab82170bc15031d093d1a31bd01cbddb78 Mon Sep 17 00:00:00 2001 From: vladimir remar Date: Fri, 20 May 2022 09:13:21 +0200 Subject: [PATCH 06/18] add minimun value to insights_lookback_window field and replace exclusiveMaximum with maximun --- .../source_facebook_marketing/spec.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index d6b363cd1439d..dce15c0246d6e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -80,7 +80,8 @@ class Config: insights_lookback_window: Optional[PositiveInt] = Field( title="Custom Insights Lookback Window", description=("The attribution window"), - exclusiveMaximum=28, + maximum=28, + mininum=1, default=28, ) @@ -167,6 +168,7 @@ class Config: title="Insights Lookback Window", order=7, description=("The attribution window"), - exclusiveMaximum=28, + maximum=28, + mininum=1, default=28, ) From 14b3007720c8fc82d560ae815feabe4b03cd32e9 Mon Sep 17 00:00:00 2001 From: vladimir remar Date: Fri, 20 May 2022 10:19:22 +0200 Subject: [PATCH 07/18] fix unit tests: test_base_insight_streams --- .../unit_tests/test_base_insight_streams.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 4ba21fc0d3417..424b1bc7b4e5b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -49,7 +49,7 @@ def async_job_mock_fixture(mocker): class TestBaseInsightsStream: def test_init(self, api): - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1)) + stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) assert not stream.breakdowns assert stream.action_breakdowns == AdsInsights.ALL_ACTION_BREAKDOWNS @@ -64,6 +64,7 @@ def test_init_override(self, api): name="CustomName", breakdowns=["test1", "test2"], action_breakdowns=["field1", "field2"], + insights_lookback_window=28, ) assert stream.breakdowns == ["test1", "test2"] @@ -83,6 +84,7 @@ def test_read_records_all(self, mocker, api): api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), + insights_lookback_window=28, ) records = list( @@ -102,11 +104,7 @@ def test_read_records_random_order(self, mocker, api): job = mocker.Mock(spec=AsyncJob) job.get_result.return_value = [mocker.Mock(), mocker.Mock(), mocker.Mock()] job.interval = pendulum.Period(pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1)) - stream = AdsInsights( - api=api, - start_date=datetime(2010, 1, 1), - end_date=datetime(2011, 1, 1), - ) + stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) records = list( stream.read_records( @@ -141,11 +139,7 @@ def test_read_records_random_order(self, mocker, api): ) def test_state(self, api, state): """State setter/getter should work with all combinations""" - stream = AdsInsights( - api=api, - start_date=datetime(2010, 1, 1), - end_date=datetime(2011, 1, 1), - ) + stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) assert stream.state == {} @@ -160,7 +154,7 @@ def test_state(self, api, state): def test_stream_slices_no_state(self, api, async_manager_mock, start_date): """Stream will use start_date when there is not state""" end_date = start_date + duration(weeks=2) - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date) + stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental)) @@ -177,7 +171,7 @@ def test_stream_slices_no_state_close_to_now(self, api, async_manager_mock, rece """Stream will use start_date when there is not state and start_date within 28d from now""" start_date = recent_start_date end_date = pendulum.now() - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date) + stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental)) @@ -195,7 +189,7 @@ def test_stream_slices_with_state(self, api, async_manager_mock, start_date): end_date = start_date + duration(days=10) cursor_value = start_date + duration(days=5) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date) + stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) @@ -214,7 +208,7 @@ def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, re end_date = pendulum.now() cursor_value = end_date - duration(days=1) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date) + stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) @@ -235,7 +229,7 @@ def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, star AdsInsights.cursor_field: cursor_value.date().isoformat(), "slices": [(cursor_value + duration(days=1)).date().isoformat(), (cursor_value + duration(days=3)).date().isoformat()], } - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date) + stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) @@ -249,7 +243,7 @@ def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, star assert generated_jobs[1].interval.start == cursor_value.date() + duration(days=4) def test_get_json_schema(self, api): - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1)) + stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) schema = stream.get_json_schema() @@ -259,7 +253,11 @@ def test_get_json_schema(self, api): def test_get_json_schema_custom(self, api): stream = AdsInsights( - api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), breakdowns=["device_platform", "country"] + api=api, + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + breakdowns=["device_platform", "country"], + insights_lookback_window=28, ) schema = stream.get_json_schema() @@ -273,6 +271,7 @@ def test_fields(self, api): api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), + insights_lookback_window=28, ) fields = stream.fields @@ -287,6 +286,7 @@ def test_fields_custom(self, api): start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], + insights_lookback_window=28, ) assert stream.fields == ["account_id", "account_currency"] From 439d527440b932ce797d176873e856b2bb56469b Mon Sep 17 00:00:00 2001 From: "vladimir.remar" Date: Mon, 23 May 2022 11:58:40 +0200 Subject: [PATCH 08/18] fix order on specs --- .../source-facebook-marketing/source_facebook_marketing/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index dce15c0246d6e..2eedd75493a28 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -166,7 +166,7 @@ class Config: insights_lookback_window: Optional[PositiveInt] = Field( title="Insights Lookback Window", - order=7, + order=8, description=("The attribution window"), maximum=28, mininum=1, From a0d13f8060434ff1b1f9b081cb2abdac6e33e57f Mon Sep 17 00:00:00 2001 From: "vladimir.remar" Date: Mon, 23 May 2022 12:01:06 +0200 Subject: [PATCH 09/18] update integration spec file --- .../integration_tests/spec.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 6113be0de55ca..3faeeed1cc6e9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -289,6 +289,14 @@ "examples": ["2017-01-26T00:00:00Z"], "type": "string", "format": "date-time" + }, + "insights_lookback_window": { + "title": "Insights Lookback Window", + "description": "The attribution window", + "default": 28, + "maximum": 28, + "minimum": 1, + "type": "integer" } }, "required": ["name"] @@ -301,6 +309,15 @@ "order": 7, "exclusiveMinimum": 0, "type": "integer" + }, + "insights_lookback_window": { + "title": "Insights Lookback Window", + "description": "The attribution window", + "default": 28, + "order": 8, + "maximum": 28, + "minimum": 1, + "type": "integer" } }, "required": ["account_id", "start_date", "access_token"] From d549143e526cb317e83846843352c64888563243 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 23 May 2022 17:51:18 +0200 Subject: [PATCH 10/18] update test spec.json --- .../source-facebook-marketing/integration_tests/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 3faeeed1cc6e9..7f5ff90066d3d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -291,7 +291,7 @@ "format": "date-time" }, "insights_lookback_window": { - "title": "Insights Lookback Window", + "title": "Custom Insights Lookback Window", "description": "The attribution window", "default": 28, "maximum": 28, From c86417f0653bde000c413b7df9eb9a6528961421 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 23 May 2022 18:32:27 +0200 Subject: [PATCH 11/18] update test spec.json --- .../source-facebook-marketing/integration_tests/spec.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 7f5ff90066d3d..338bbe162a276 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -295,7 +295,8 @@ "description": "The attribution window", "default": 28, "maximum": 28, - "minimum": 1, + "mininum": 1, + "exclusiveMinimum": 0, "type": "integer" } }, @@ -316,7 +317,8 @@ "default": 28, "order": 8, "maximum": 28, - "minimum": 1, + "mininum": 1, + "exclusiveMinimum": 0, "type": "integer" } }, From 246c2dd4dfc2153b9a1bb9bff92030ef7c690167 Mon Sep 17 00:00:00 2001 From: "vladimir.remar" Date: Tue, 24 May 2022 15:42:26 +0200 Subject: [PATCH 12/18] update refresh date --- .../source_facebook_marketing/streams/base_insight_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index dacc9c7bc798f..ca3e998ad158b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -196,7 +196,7 @@ def _generate_async_jobs(self, params: Mapping) -> Iterator[AsyncJob]: """ today = pendulum.today(tz="UTC").date() - refresh_date = today - self.INSIGHTS_LOOKBACK_PERIOD + refresh_date = today - self.insights_lookback_period for ts_start in self._date_intervals(): if ts_start in self._completed_slices: From b39262ab29dbc6db82252e677d987dcedaaf3727 Mon Sep 17 00:00:00 2001 From: "vladimir.remar" Date: Tue, 24 May 2022 15:43:37 +0200 Subject: [PATCH 13/18] update test_base_insight_streams --- .../unit_tests/test_base_insight_streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 262641082df80..65b8673d3ff5c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -297,7 +297,7 @@ def test_completed_slices_in_lookback_period(self, api, monkeypatch, set_today): start_date = pendulum.parse("2020-03-01") end_date = pendulum.parse("2020-05-01") set_today("2020-04-01") - monkeypatch.setattr(AdsInsights, "INSIGHTS_LOOKBACK_PERIOD", pendulum.duration(days=10)) + monkeypatch.setattr(AdsInsights, "insights_lookback_period", pendulum.duration(days=10)) monkeypatch.setattr(source_facebook_marketing.streams.base_insight_streams, "InsightAsyncJob", FakeInsightAsyncJob) monkeypatch.setattr(source_facebook_marketing.streams.base_insight_streams, "InsightAsyncJobManager", FakeInsightAsyncJobManager) @@ -327,7 +327,7 @@ def test_incremental_lookback_period_updated(self, api, monkeypatch, set_today): start_date = pendulum.parse("2020-03-01") end_date = pendulum.parse("2020-05-01") yesterday, _ = set_today("2020-04-01") - monkeypatch.setattr(AdsInsights, "INSIGHTS_LOOKBACK_PERIOD", pendulum.duration(days=20)) + monkeypatch.setattr(AdsInsights, "insights_lookback_period", pendulum.duration(days=20)) monkeypatch.setattr(source_facebook_marketing.streams.base_insight_streams, "InsightAsyncJob", FakeInsightAsyncJob) monkeypatch.setattr(source_facebook_marketing.streams.base_insight_streams, "InsightAsyncJobManager", FakeInsightAsyncJobManager) From a6577ed1c4b6ebdb753a6a77858d24626f2d683f Mon Sep 17 00:00:00 2001 From: vladimir remar Date: Wed, 25 May 2022 09:36:46 +0200 Subject: [PATCH 14/18] remove round brackets from insights_lookback_window description --- .../source_facebook_marketing/spec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 2eedd75493a28..32ac4eeb47d61 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -79,7 +79,7 @@ class Config: ) insights_lookback_window: Optional[PositiveInt] = Field( title="Custom Insights Lookback Window", - description=("The attribution window"), + description="The attribution window", maximum=28, mininum=1, default=28, @@ -167,7 +167,7 @@ class Config: insights_lookback_window: Optional[PositiveInt] = Field( title="Insights Lookback Window", order=8, - description=("The attribution window"), + description="The attribution window", maximum=28, mininum=1, default=28, From 4524fc4206bad4b7ba62acf3e9fb9e2e02709af6 Mon Sep 17 00:00:00 2001 From: vladimir remar Date: Wed, 25 May 2022 09:39:04 +0200 Subject: [PATCH 15/18] remove monkeypatch for AdsInsights in test_incremental_lookback_period_updated and test_completed_slices_in_lookback_period --- .../unit_tests/test_base_insight_streams.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 65b8673d3ff5c..efe8a2ae8bfc8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -297,7 +297,7 @@ def test_completed_slices_in_lookback_period(self, api, monkeypatch, set_today): start_date = pendulum.parse("2020-03-01") end_date = pendulum.parse("2020-05-01") set_today("2020-04-01") - monkeypatch.setattr(AdsInsights, "insights_lookback_period", pendulum.duration(days=10)) + monkeypatch.setattr(source_facebook_marketing.streams.base_insight_streams, "InsightAsyncJob", FakeInsightAsyncJob) monkeypatch.setattr(source_facebook_marketing.streams.base_insight_streams, "InsightAsyncJobManager", FakeInsightAsyncJobManager) @@ -311,7 +311,7 @@ def test_completed_slices_in_lookback_period(self, api, monkeypatch, set_today): "time_increment": 1, } - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date) + stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=10) stream.state = state assert stream._completed_slices == {pendulum.Date(2020, 3, 21), pendulum.Date(2020, 3, 22), pendulum.Date(2020, 3, 23)} @@ -327,11 +327,11 @@ def test_incremental_lookback_period_updated(self, api, monkeypatch, set_today): start_date = pendulum.parse("2020-03-01") end_date = pendulum.parse("2020-05-01") yesterday, _ = set_today("2020-04-01") - monkeypatch.setattr(AdsInsights, "insights_lookback_period", pendulum.duration(days=20)) + monkeypatch.setattr(source_facebook_marketing.streams.base_insight_streams, "InsightAsyncJob", FakeInsightAsyncJob) monkeypatch.setattr(source_facebook_marketing.streams.base_insight_streams, "InsightAsyncJobManager", FakeInsightAsyncJobManager) - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date) + stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=20) records = read_full_refresh(stream) assert len(records) == (yesterday - start_date).days + 1 From fe1b30a3005ddfed082a4b282eac7014f8afef16 Mon Sep 17 00:00:00 2001 From: "vladimir.remar" Date: Mon, 30 May 2022 09:00:50 +0200 Subject: [PATCH 16/18] update connector version in Dockerfile --- .../connectors/source-facebook-marketing/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 6568229505083..9dafcb4007be9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.49 +LABEL io.airbyte.version=0.2.50 LABEL io.airbyte.name=airbyte/source-facebook-marketing From b7d7973cf70563559a494c4bca516b5f99d33396 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 30 May 2022 09:06:01 +0200 Subject: [PATCH 17/18] fix typo in changelog --- docs/integrations/sources/facebook-marketing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 16a9fad611ec5..6c378b8fcd648 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -108,7 +108,7 @@ For more information, see the [Facebook Insights API documentation.](https://dev | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.2.50 | 2022-04-27 | [12402](https://github.com/airbytehq/airbyte/pull/12402) | Add lookback window to insgiths streams | +| 0.2.50 | 2022-04-27 | [12402](https://github.com/airbytehq/airbyte/pull/12402) | Add lookback window to insights streams | | 0.2.49 | 2022-05-20 | [13047](https://github.com/airbytehq/airbyte/pull/13047) | Fix duplicating records during insights lookback period | | 0.2.48 | 2022-05-19 | [13008](https://github.com/airbytehq/airbyte/pull/13008) | Update CDK to v0.1.58 avoid crashing on incorrect stream schemas | | 0.2.47 | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions | From 7483176cb9abbbdfd87e4b6c257c014277a13fdd Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 31 May 2022 07:31:24 +0000 Subject: [PATCH 18/18] auto-bump connector version --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 3e02e635e9ef7..8c9ac446e5502 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -240,7 +240,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.49 + dockerImageTag: 0.2.50 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 0ee0b7d60cd9a..fbccfd14538cf 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1838,7 +1838,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.49" +- dockerImage: "airbyte/source-facebook-marketing:0.2.50" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" @@ -2141,6 +2141,14 @@ - "2017-01-26T00:00:00Z" type: "string" format: "date-time" + insights_lookback_window: + title: "Custom Insights Lookback Window" + description: "The attribution window" + default: 28 + maximum: 28 + mininum: 1 + exclusiveMinimum: 0 + type: "integer" required: - "name" page_size: @@ -2153,6 +2161,15 @@ order: 7 exclusiveMinimum: 0 type: "integer" + insights_lookback_window: + title: "Insights Lookback Window" + description: "The attribution window" + default: 28 + order: 8 + maximum: 28 + mininum: 1 + exclusiveMinimum: 0 + type: "integer" required: - "account_id" - "start_date"