diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index e61cc18335b3b..1c2b842a4cd52 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.1.0 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 21f4a862c9e9f..0324ab907136c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -285,6 +285,13 @@ ] } }, + "action_report_time": { + "title": "Action Report Time", + "description": "Determines the report time of action stats. For example, if a person saw the ad on Jan 1st but converted on Jan 2nd, when you query the API with action_report_time=impression, you see a conversion on Jan 1st. When you query the API with action_report_time=conversion, you see a conversion on Jan 2nd.", + "default": "mixed", + "enum": ["conversion", "impression", "mixed"], + "type": "string" + }, "time_increment": { "title": "Time Increment", "description": "Time window in days by which to aggregate statistics. The sync will be chunked into N day intervals, where N is the number of days you specified. For example, if you set this value to 7, then all statistics will be reported as 7-day aggregates by starting from the start_date. If the start and end dates are October 1st and October 30th, then the connector will output 5 records: 01 - 06, 07 - 13, 14 - 20, 21 - 27, and 28 - 30 (3 days only).", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index d2795b64131da..b80b804344831 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c - dockerImageTag: 1.0.1 + dockerImageTag: 1.1.0 dockerRepository: airbyte/source-facebook-marketing githubIssueLabel: source-facebook-marketing icon: facebook.svg diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index a4c679fecc9ac..a5cbe1aa93cd8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -261,6 +261,7 @@ def get_custom_insights_streams(self, api: API, config: ConnectorConfig) -> List breakdowns=list(set(insight.breakdowns)), action_breakdowns=list(set(insight.action_breakdowns)), action_breakdowns_allow_empty=config.action_breakdowns_allow_empty, + action_report_time=insight.action_report_time, time_increment=insight.time_increment, start_date=insight.start_date or config.start_date, end_date=insight.end_date or config.end_date, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index b9c0f9a51fc62..901451dfffbc9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -52,6 +52,17 @@ class Config: default=[], ) + action_report_time: str = Field( + title="Action Report Time", + description=( + "Determines the report time of action stats. For example, if a person saw the ad on Jan 1st " + "but converted on Jan 2nd, when you query the API with action_report_time=impression, you see a conversion on Jan 1st. " + "When you query the API with action_report_time=conversion, you see a conversion on Jan 2nd." + ), + default="mixed", + enum=["conversion", "impression", "mixed"], + ) + time_increment: Optional[PositiveInt] = Field( title="Time Increment", description=( diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 8e93929c9c33d..5913b8a78eb47 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -60,6 +60,7 @@ def __init__( breakdowns: List[str] = None, action_breakdowns: List[str] = None, action_breakdowns_allow_empty: bool = False, + action_report_time: str = "mixed", time_increment: Optional[int] = None, insights_lookback_window: int = None, level: str = "ad", @@ -78,6 +79,7 @@ def __init__( if breakdowns is not None: self.breakdowns = breakdowns self.time_increment = time_increment or self.time_increment + self.action_report_time = action_report_time self._new_class_name = name self._insights_lookback_window = insights_lookback_window self.level = level @@ -276,6 +278,7 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]: return { "level": self.level, "action_breakdowns": self.action_breakdowns, + "action_report_time": self.action_report_time, "breakdowns": self.breakdowns, "fields": self.fields, "time_increment": self.time_increment, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py index 9fa467b9d6e8c..69941a42501d5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py @@ -40,6 +40,7 @@ def job_fixture(api, account): params = { "level": "ad", "action_breakdowns": [], + "action_report_time": "mixed", "breakdowns": [], "fields": ["field1", "field2"], "time_increment": 1, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 59952d4e915ec..c773ad505deca 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -55,6 +55,7 @@ def test_init(self, api): assert stream.action_breakdowns == ["action_type", "action_target_id", "action_destination"] assert stream.name == "ads_insights" assert stream.primary_key == ["date_start", "account_id", "ad_id"] + assert stream.action_report_time == "mixed" def test_init_override(self, api): stream = AdsInsights( diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py index 5fccdaf5f6e99..13638dbfef97f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py @@ -49,10 +49,12 @@ def test_filter_all_statuses(api, mocker): @pytest.mark.parametrize( - "url", ["https://graph.facebook.com", "https://graph.facebook.com?test=123%23%24%25%2A&test2=456", "https://graph.facebook.com?"] + "url", ["https://graph.facebook.com", + "https://graph.facebook.com?test=123%23%24%25%2A&test2=456", "https://graph.facebook.com?"] ) def test_fetch_thumbnail_data_url(url, requests_mock): - requests_mock.get(url, status_code=200, headers={"content-type": "content-type"}, content=b"") + requests_mock.get(url, status_code=200, headers={ + "content-type": "content-type"}, content=b"") assert fetch_thumbnail_data_url(url) == "data:content-type;base64," @@ -61,7 +63,8 @@ def test_parse_call_rate_header(): "x-business-use-case-usage": '{"test":[{"type":"ads_management","call_count":1,"total_cputime":1,' '"total_time":1,"estimated_time_to_regain_access":1}]}' } - assert MyFacebookAdsApi._parse_call_rate_header(headers) == (1, duration(minutes=1)) + assert MyFacebookAdsApi._parse_call_rate_header( + headers) == (1, duration(minutes=1)) @pytest.mark.parametrize( @@ -69,30 +72,53 @@ def test_parse_call_rate_header(): [ [AdsInsights, [], ["action_type", "action_target_id", "action_destination"]], [AdsInsightsActionType, [], ["action_type"]], - [AdsInsightsAgeAndGender, ["age", "gender"], ["action_type", "action_target_id", "action_destination"]], - [AdsInsightsCountry, ["country"], ["action_type", "action_target_id", "action_destination"]], - [AdsInsightsDma, ["dma"], ["action_type", "action_target_id", "action_destination"]], - [AdsInsightsPlatformAndDevice, ["publisher_platform", "platform_position", "impression_device"], ["action_type"]], - [AdsInsightsRegion, ["region"], ["action_type", "action_target_id", "action_destination"]], + [AdsInsightsAgeAndGender, ["age", "gender"], [ + "action_type", "action_target_id", "action_destination"]], + [AdsInsightsCountry, ["country"], ["action_type", + "action_target_id", "action_destination"]], + [AdsInsightsDma, ["dma"], ["action_type", + "action_target_id", "action_destination"]], + [AdsInsightsPlatformAndDevice, ["publisher_platform", + "platform_position", "impression_device"], ["action_type"]], + [AdsInsightsRegion, ["region"], ["action_type", + "action_target_id", "action_destination"]], ], ) def test_ads_insights_breakdowns(class_name, breakdowns, action_breakdowns): - kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1} + kwargs = {"api": None, "start_date": pendulum.now( + ), "end_date": pendulum.now(), "insights_lookback_window": 1} stream = class_name(**kwargs) assert stream.breakdowns == breakdowns assert stream.action_breakdowns == action_breakdowns def test_custom_ads_insights_breakdowns(): - kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1} - stream = AdsInsights(breakdowns=["mmm"], action_breakdowns=["action_destination"], **kwargs) + kwargs = {"api": None, "start_date": pendulum.now( + ), "end_date": pendulum.now(), "insights_lookback_window": 1} + stream = AdsInsights(breakdowns=["mmm"], action_breakdowns=[ + "action_destination"], **kwargs) assert stream.breakdowns == ["mmm"] assert stream.action_breakdowns == ["action_destination"] stream = AdsInsights(breakdowns=[], action_breakdowns=[], **kwargs) assert stream.breakdowns == [] - assert stream.action_breakdowns == ["action_type", "action_target_id", "action_destination"] + assert stream.action_breakdowns == [ + "action_type", "action_target_id", "action_destination"] - stream = AdsInsights(breakdowns=[], action_breakdowns=[], action_breakdowns_allow_empty=True, **kwargs) + stream = AdsInsights(breakdowns=[], action_breakdowns=[], + action_breakdowns_allow_empty=True, **kwargs) assert stream.breakdowns == [] assert stream.action_breakdowns == [] + + +def test_custom_ads_insights_action_report_times(): + kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now( + ), "insights_lookback_window": 1, "action_breakdowns": ["action_destination"], "breakdowns": []} + stream = AdsInsights(**kwargs) + assert stream.action_report_time == "mixed" + + stream = AdsInsights(action_report_time="conversion", **kwargs) + assert stream.action_report_time == "conversion" + + stream = AdsInsights(action_report_time="impression", **kwargs) + assert stream.action_report_time == "impression" diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 163330a7ff0e3..41973ba1d04c2 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -43,27 +43,28 @@ This page guides you through the process of setting up the Facebook Marketing so 8. (Optional) Toggle the **Fetch Thumbnail Images** button to fetch the `thumbnail_url` and store the result in `thumbnail_data_url` for each [Ad Creative](https://developers.facebook.com/docs/marketing-api/creative/). 9. (Optional) In the Custom Insights section. A list which contains ad statistics entries, each entry must have a name and can contain fields, breakdowns or action_breakdowns. Click on "add" to fill this field. - To retrieve specific fields from Facebook Ads Insights combined with other breakdowns, you can choose which fields and breakdowns to sync. - We recommend following the Facebook Marketing [documentation](https://developers.facebook.com/docs/marketing-api/insights/breakdowns) to understand the breakdown limitations. Some fields can not be requested and many others only work when combined with specific fields. For example, the breakdown `app_id` is only supported with the `total_postbacks` field. - - To configure Custom Insights: - - 1. For **Name**, enter a name for the insight. This will be used as the Airbyte stream name - 2. For **Level**, enter the level of the fields you want to pull from the Facebook Marketing API. By default, 'ad'. You can specify also account, campaign or adset. - 3. For **Fields**, enter a list of the fields you want to pull from the Facebook Marketing API. - 4. For **End Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and before this date will be replicated. If this field is blank, Airbyte will replicate the latest data. - 5. For **Breakdowns**, enter a list of the breakdowns you want to configure. - 6. For **Start Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. - 7. For **Action Breakdown**, enter a list of the action breakdowns you want to configure. - 8. For **Custom Insights Lookback Window**, fill in the appropriate value. See [more](#facebook-marketing-attribution-reporting) on this parameter. - 9. Click **Done**. - 10. For **Page Size of Requests**, fill in the page size in case pagination kicks in. Feel free to ignore it, the default value should work in most cases. - 11. For **Insights Lookback Window**, fill in the appropriate value. Facebook freezes insight data 28 days after it was generated, which means that all data from the past 28 days may have changed since we last emitted it, so you can retrieve refreshed insights from the past by setting this parameter. If you set a custom lookback window value in Facebook account, please provide the same value here. See [more](#facebook-marketing-attribution-reporting) on this parameter. - 12. Click **Set up source**. - - :::warning - Additional streams for Facebook Marketing are dynamically created based on the specified Custom Insights. For an existing Facebook Marketing source, when you are updating or removing Custom Insights, you should also ensure that any connections syncing to these streams are either disabled or have had their source schema refreshed. - ::: + To retrieve specific fields from Facebook Ads Insights combined with other breakdowns, you can choose which fields and breakdowns to sync. + We recommend following the Facebook Marketing [documentation](https://developers.facebook.com/docs/marketing-api/insights/breakdowns) to understand the breakdown limitations. Some fields can not be requested and many others only work when combined with specific fields. For example, the breakdown `app_id` is only supported with the `total_postbacks` field. + + To configure Custom Insights: + + 1. For **Name**, enter a name for the insight. This will be used as the Airbyte stream name + 2. For **Level**, enter the level of the fields you want to pull from the Facebook Marketing API. By default, 'ad'. You can specify also account, campaign or adset. + 3. For **Fields**, enter a list of the fields you want to pull from the Facebook Marketing API. + 4. For **End Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and before this date will be replicated. If this field is blank, Airbyte will replicate the latest data. + 5. For **Breakdowns**, enter a list of the breakdowns you want to configure. + 6. For **Start Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. + 7. For **Action Breakdown**, enter a list of the action breakdowns you want to configure. + 8. For **Action Report Time**, enter the action report time you want to configure (mixed, conversion or impression). + 9. For **Custom Insights Lookback Window**, fill in the appropriate value. See [more](#facebook-marketing-attribution-reporting) on this parameter. + 10. Click **Done**. + 12. For **Page Size of Requests**, fill in the page size in case pagination kicks in. Feel free to ignore it, the default value should work in most cases. + 13. For **Insights Lookback Window**, fill in the appropriate value. Facebook freezes insight data 28 days after it was generated, which means that all data from the past 28 days may have changed since we last emitted it, so you can retrieve refreshed insights from the past by setting this parameter. If you set a custom lookback window value in Facebook account, please provide the same value here. See [more](#facebook-marketing-attribution-reporting) on this parameter. + 14. Click **Set up source**. + + :::warning + Additional streams for Facebook Marketing are dynamically created based on the specified Custom Insights. For an existing Facebook Marketing source, when you are updating or removing Custom Insights, you should also ensure that any connections syncing to these streams are either disabled or have had their source schema refreshed. + ::: @@ -166,7 +167,8 @@ Please be informed that the connector uses the `lookback_window` parameter to pe ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.1.0 | 2023-07-11 | [26345](https://github.com/airbytehq/airbyte/pull/26345) | add new `action_report_time` attribute to `AdInsights` class | | 1.0.1 | 2023-07-07 | [27979](https://github.com/airbytehq/airbyte/pull/27979) | Added the ability to restore the reduced request record limit after the successful retry, and handle the `unknown error` (code 99) with the retry strategy | | 1.0.0 | 2023-07-05 | [27563](https://github.com/airbytehq/airbyte/pull/27563) | Migrate to FB SDK version 17 | | 0.5.0 | 2023-06-26 | [27728](https://github.com/airbytehq/airbyte/pull/27728) | License Update: Elv2 |